Downloading crates ...
  Downloaded rand_xorshift v0.3.0
  Downloaded bit-set v0.5.3
  Downloaded similar v2.2.1
  Downloaded subprocess v0.2.9
  Downloaded quick-error v1.2.3
  Downloaded wait-timeout v0.2.0
  Downloaded structmeta v0.2.0
  Downloaded tokio-test v0.4.2
  Downloaded unarray v0.1.4
  Downloaded structmeta-derive v0.2.0
  Downloaded httptest v0.15.4
  Downloaded convert_case v0.5.0
  Downloaded tokio-stream v0.1.8
  Downloaded test-strategy v0.3.1
  Downloaded regex-automata v0.1.10
  Downloaded expectorate v1.0.7
  Downloaded proptest v1.2.0
  Downloaded bstr v0.2.17
  Downloaded bit-vec v0.6.3
  Downloaded newline-converter v0.3.0
  Downloaded rusty-fork v0.3.0
   Compiling libc v0.2.147
   Compiling proc-macro2 v1.0.63
   Compiling quote v1.0.29
   Compiling unicode-ident v1.0.11
   Compiling cfg-if v1.0.0
   Compiling autocfg v1.1.0
   Compiling serde v1.0.167
   Compiling version_check v0.9.4
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libc-0.2.147/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="extra_traits"' --cfg 'feature="std"' -C metadata=b5c3e22808162f2b -C extra-filename=-b5c3e22808162f2b --out-dir /work/oxidecomputer/crucible/target/debug/build/libc-b5c3e22808162f2b -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/proc-macro2-1.0.63/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="proc-macro"' -C metadata=9a0ae810f5602a16 -C extra-filename=-9a0ae810f5602a16 --out-dir /work/oxidecomputer/crucible/target/debug/build/proc-macro2-9a0ae810f5602a16 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/quote-1.0.29/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="proc-macro"' -C metadata=e1ca2b57372da817 -C extra-filename=-e1ca2b57372da817 --out-dir /work/oxidecomputer/crucible/target/debug/build/quote-e1ca2b57372da817 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name unicode_ident --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-ident-1.0.11/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=81e7752fff89e70f -C extra-filename=-81e7752fff89e70f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name cfg_if --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/cfg-if-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=a93276574f0edf39 -C extra-filename=-a93276574f0edf39 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde-1.0.167/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="rc"' --cfg 'feature="serde_derive"' --cfg 'feature="std"' -C metadata=a519c991a68c99af -C extra-filename=-a519c991a68c99af --out-dir /work/oxidecomputer/crucible/target/debug/build/serde-a519c991a68c99af -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name autocfg /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/autocfg-1.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=b1cec8cc882d2fdd -C extra-filename=-b1cec8cc882d2fdd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name version_check /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/version_check-0.9.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=23142ed5bf6178a0 -C extra-filename=-23142ed5bf6178a0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling once_cell v1.14.0
     Running `rustc --crate-name once_cell --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/once_cell-1.14.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="race"' --cfg 'feature="std"' --cfg 'feature="unstable"' -C metadata=fdb5f9e769d1e589 -C extra-filename=-fdb5f9e769d1e589 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling syn v1.0.107
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/syn-1.0.107/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="clone-impls"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="extra-traits"' --cfg 'feature="full"' --cfg 'feature="parsing"' --cfg 'feature="printing"' --cfg 'feature="proc-macro"' --cfg 'feature="quote"' --cfg 'feature="visit"' --cfg 'feature="visit-mut"' -C metadata=52840f115412428f -C extra-filename=-52840f115412428f --out-dir /work/oxidecomputer/crucible/target/debug/build/syn-52840f115412428f -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/quote-e1ca2b57372da817/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/serde-a519c991a68c99af/build-script-build`
   Compiling log v0.4.14
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/log-0.4.14/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="std"' -C metadata=9aabd99961801a55 -C extra-filename=-9aabd99961801a55 --out-dir /work/oxidecomputer/crucible/target/debug/build/log-9aabd99961801a55 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/proc-macro2-9a0ae810f5602a16/build-script-build`
   Compiling itoa v1.0.1
     Running `rustc --crate-name itoa --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/itoa-1.0.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f0781104e344570e -C extra-filename=-f0781104e344570e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/libc-b5c3e22808162f2b/build-script-build`
   Compiling memchr v2.5.0
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/memchr-2.5.0/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=39ec00add1e73006 -C extra-filename=-39ec00add1e73006 --out-dir /work/oxidecomputer/crucible/target/debug/build/memchr-39ec00add1e73006 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling libm v0.2.6
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libm-0.2.6/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' -C metadata=80351c6c420eb23d -C extra-filename=-80351c6c420eb23d --out-dir /work/oxidecomputer/crucible/target/debug/build/libm-80351c6c420eb23d -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling num-traits v0.2.16
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-traits-0.2.16/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="libm"' --cfg 'feature="std"' -C metadata=0ea88a0660a9c2fb -C extra-filename=-0ea88a0660a9c2fb --out-dir /work/oxidecomputer/crucible/target/debug/build/num-traits-0ea88a0660a9c2fb -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
     Running `rustc --crate-name proc_macro2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/proc-macro2-1.0.63/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="proc-macro"' -C metadata=a0e1129b171da08c -C extra-filename=-a0e1129b171da08c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern unicode_ident=/work/oxidecomputer/crucible/target/debug/deps/libunicode_ident-81e7752fff89e70f.rmeta --cap-lints allow --cfg wrap_proc_macro`
     Running `rustc --crate-name libc /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libc-0.2.147/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="extra_traits"' --cfg 'feature="std"' -C metadata=a748caf4ceff51bd -C extra-filename=-a748caf4ceff51bd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg freebsd11 --cfg libc_priv_mod_use --cfg libc_union --cfg libc_const_size_of --cfg libc_align --cfg libc_int128 --cfg libc_core_cvoid --cfg libc_packedN --cfg libc_cfg_target_vendor --cfg libc_non_exhaustive --cfg libc_long_array --cfg libc_ptr_addr_of --cfg libc_underscore_const_names --cfg libc_const_extern_fn`
     Running `/work/oxidecomputer/crucible/target/debug/build/syn-52840f115412428f/build-script-build`
   Compiling scopeguard v1.1.0
     Running `rustc --crate-name scopeguard /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/scopeguard-1.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=00b7ece4eb7b8e7e -C extra-filename=-00b7ece4eb7b8e7e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling pin-project-lite v0.2.13
     Running `rustc --crate-name pin_project_lite --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pin-project-lite-0.2.13/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c035e320730303c2 -C extra-filename=-c035e320730303c2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/log-9aabd99961801a55/build-script-build`
     Running `rustc --crate-name log /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/log-0.4.14/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=930e200cffaa7930 -C extra-filename=-930e200cffaa7930 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --cap-lints allow --cfg atomic_cas --cfg has_atomics`
   Compiling smallvec v1.10.0
     Running `rustc --crate-name smallvec --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/smallvec-1.10.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=397f26bd8c84e528 -C extra-filename=-397f26bd8c84e528 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/num-traits-0ea88a0660a9c2fb/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/libm-80351c6c420eb23d/build-script-build`
   Compiling thiserror v1.0.40
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thiserror-1.0.40/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=23fb1ce510a925d0 -C extra-filename=-23fb1ce510a925d0 --out-dir /work/oxidecomputer/crucible/target/debug/build/thiserror-23fb1ce510a925d0 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name libm --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libm-0.2.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=e5a688f05412e317 -C extra-filename=-e5a688f05412e317 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/memchr-39ec00add1e73006/build-script-build`
     Running `rustc --crate-name memchr --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/memchr-2.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=9611c546f9b73cca -C extra-filename=-9611c546f9b73cca --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg memchr_runtime_simd --cfg memchr_runtime_sse2 --cfg memchr_runtime_sse42 --cfg memchr_runtime_avx`
   Compiling ahash v0.7.6
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ahash-0.7.6/./build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=72382e4025e5c1c3 -C extra-filename=-72382e4025e5c1c3 --out-dir /work/oxidecomputer/crucible/target/debug/build/ahash-72382e4025e5c1c3 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern version_check=/work/oxidecomputer/crucible/target/debug/deps/libversion_check-23142ed5bf6178a0.rlib --cap-lints allow`
     Running `rustc --crate-name quote --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/quote-1.0.29/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="proc-macro"' -C metadata=2ffbaa80dd156621 -C extra-filename=-2ffbaa80dd156621 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/thiserror-23fb1ce510a925d0/build-script-build`
   Compiling parking_lot_core v0.9.1
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/parking_lot_core-0.9.1/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=309b532cd044ced9 -C extra-filename=-309b532cd044ced9 --out-dir /work/oxidecomputer/crucible/target/debug/build/parking_lot_core-309b532cd044ced9 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling getrandom v0.2.5
     Running `rustc --crate-name getrandom --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/getrandom-0.2.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=567199de146d617e -C extra-filename=-567199de146d617e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling syn v2.0.23
     Running `rustc --crate-name syn --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/syn-2.0.23/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="clone-impls"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="extra-traits"' --cfg 'feature="full"' --cfg 'feature="parsing"' --cfg 'feature="printing"' --cfg 'feature="proc-macro"' --cfg 'feature="quote"' --cfg 'feature="visit"' --cfg 'feature="visit-mut"' -C metadata=baedf68a9175a6da -C extra-filename=-baedf68a9175a6da --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern unicode_ident=/work/oxidecomputer/crucible/target/debug/deps/libunicode_ident-81e7752fff89e70f.rmeta --cap-lints allow`
     Running `rustc --crate-name syn --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/syn-1.0.107/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="clone-impls"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="extra-traits"' --cfg 'feature="full"' --cfg 'feature="parsing"' --cfg 'feature="printing"' --cfg 'feature="proc-macro"' --cfg 'feature="quote"' --cfg 'feature="visit"' --cfg 'feature="visit-mut"' -C metadata=837f9a049f77ca38 -C extra-filename=-837f9a049f77ca38 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern unicode_ident=/work/oxidecomputer/crucible/target/debug/deps/libunicode_ident-81e7752fff89e70f.rmeta --cap-lints allow --cfg syn_disable_nightly_tests`
   Compiling jobserver v0.1.25
     Running `rustc --crate-name jobserver --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/jobserver-0.1.25/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=0ee11fba78dd3235 -C extra-filename=-0ee11fba78dd3235 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/parking_lot_core-309b532cd044ced9/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/ahash-72382e4025e5c1c3/build-script-build`
   Compiling num_cpus v1.13.1
     Running `rustc --crate-name num_cpus /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num_cpus-1.13.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=67a451bebfcc5086 -C extra-filename=-67a451bebfcc5086 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
     Running `rustc --crate-name num_traits --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-traits-0.2.16/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="libm"' --cfg 'feature="std"' -C metadata=8e50de91aba3f8f9 -C extra-filename=-8e50de91aba3f8f9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libm=/work/oxidecomputer/crucible/target/debug/deps/liblibm-e5a688f05412e317.rmeta --cap-lints allow --cfg has_to_int_unchecked --cfg has_reverse_bits --cfg has_leading_trailing_ones --cfg has_div_euclid --cfg has_copysign --cfg has_is_subnormal --cfg has_int_to_from_bytes --cfg has_float_to_from_bytes`
     Running `rustc --crate-name ahash --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ahash-0.7.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c781b32f1fcd8d92 -C extra-filename=-c781b32f1fcd8d92 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern getrandom=/work/oxidecomputer/crucible/target/debug/deps/libgetrandom-567199de146d617e.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --cap-lints allow --cfg 'feature="runtime-rng"' --cfg 'feature="folded_multiply"'`
     Running `rustc --crate-name parking_lot_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/parking_lot_core-0.9.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f45d0642d52c20c1 -C extra-filename=-f45d0642d52c20c1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern smallvec=/work/oxidecomputer/crucible/target/debug/deps/libsmallvec-397f26bd8c84e528.rmeta --cap-lints allow`
   Compiling lock_api v0.4.6
     Running `rustc --crate-name lock_api --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/lock_api-0.4.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4b01d37c549347e9 -C extra-filename=-4b01d37c549347e9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern scopeguard=/work/oxidecomputer/crucible/target/debug/deps/libscopeguard-00b7ece4eb7b8e7e.rmeta --cap-lints allow`
   Compiling cc v1.0.73
     Running `rustc --crate-name cc --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/cc-1.0.73/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="jobserver"' --cfg 'feature="parallel"' -C metadata=2976d4b8f46fa671 -C extra-filename=-2976d4b8f46fa671 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern jobserver=/work/oxidecomputer/crucible/target/debug/deps/libjobserver-0ee11fba78dd3235.rmeta --cap-lints allow`
   Compiling ryu v1.0.9
   Compiling futures-core v0.3.28
     Running `rustc --crate-name ryu --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ryu-1.0.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=062e5ac4087417b3 -C extra-filename=-062e5ac4087417b3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.28/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=f9a38619a6c6491e -C extra-filename=-f9a38619a6c6491e --out-dir /work/oxidecomputer/crucible/target/debug/build/futures-core-f9a38619a6c6491e -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling parking_lot v0.12.0
     Running `rustc --crate-name parking_lot --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/parking_lot-0.12.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="send_guard"' -C metadata=970d5c0acece447c -C extra-filename=-970d5c0acece447c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern lock_api=/work/oxidecomputer/crucible/target/debug/deps/liblock_api-4b01d37c549347e9.rmeta --extern parking_lot_core=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot_core-f45d0642d52c20c1.rmeta --cap-lints allow`
   Compiling hashbrown v0.12.3
     Running `rustc --crate-name hashbrown --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hashbrown-0.12.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="ahash"' --cfg 'feature="default"' --cfg 'feature="inline-more"' --cfg 'feature="raw"' -C metadata=3ad6614047c487f9 -C extra-filename=-3ad6614047c487f9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ahash=/work/oxidecomputer/crucible/target/debug/deps/libahash-c781b32f1fcd8d92.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/futures-core-f9a38619a6c6491e/build-script-build`
   Compiling serde_json v1.0.105
     Running `rustc --crate-name build_script_build --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.105/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=9a7f1bd5890a4c18 -C extra-filename=-9a7f1bd5890a4c18 --out-dir /work/oxidecomputer/crucible/target/debug/build/serde_json-9a7f1bd5890a4c18 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name futures_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=46c6e3a1b3966417 -C extra-filename=-46c6e3a1b3966417 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling bitflags v1.3.2
     Running `rustc --crate-name bitflags --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bitflags-1.3.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=b5bc15fb96af61fc -C extra-filename=-b5bc15fb96af61fc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling pin-utils v0.1.0
     Running `rustc --crate-name pin_utils --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pin-utils-0.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bcfb754cd1ab8c67 -C extra-filename=-bcfb754cd1ab8c67 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/serde_json-9a7f1bd5890a4c18/build-script-build`
   Compiling pkg-config v0.3.24
     Running `rustc --crate-name pkg_config /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pkg-config-0.3.24/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=e05c47386f4bdcc0 -C extra-filename=-e05c47386f4bdcc0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling signal-hook-registry v1.4.0
     Running `rustc --crate-name signal_hook_registry /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/signal-hook-registry-1.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4d955479f235827e -C extra-filename=-4d955479f235827e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling rand_core v0.6.4
     Running `rustc --crate-name rand_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_core-0.6.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="getrandom"' --cfg 'feature="std"' -C metadata=e2870cc0e01c33c9 -C extra-filename=-e2870cc0e01c33c9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern getrandom=/work/oxidecomputer/crucible/target/debug/deps/libgetrandom-567199de146d617e.rmeta --cap-lints allow`
   Compiling mio v0.8.8
     Running `rustc --crate-name mio --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mio-0.8.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="log"' --cfg 'feature="net"' --cfg 'feature="os-ext"' --cfg 'feature="os-poll"' -C metadata=27a8136cf12de2bb -C extra-filename=-27a8136cf12de2bb --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --cap-lints allow`
   Compiling futures-task v0.3.28
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-task-0.3.28/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="alloc"' --cfg 'feature="std"' -C metadata=6662546d276b2bb5 -C extra-filename=-6662546d276b2bb5 --out-dir /work/oxidecomputer/crucible/target/debug/build/futures-task-6662546d276b2bb5 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling futures-sink v0.3.28
     Running `rustc --crate-name futures_sink --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-sink-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=a6d6ed8a846c5f8a -C extra-filename=-a6d6ed8a846c5f8a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling futures-channel v0.3.28
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-channel-0.3.28/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="futures-sink"' --cfg 'feature="sink"' --cfg 'feature="std"' -C metadata=224ac16fb4ca2e60 -C extra-filename=-224ac16fb4ca2e60 --out-dir /work/oxidecomputer/crucible/target/debug/build/futures-channel-224ac16fb4ca2e60 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling fnv v1.0.7
     Running `rustc --crate-name fnv /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fnv-1.0.7/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=5641130f60a8056b -C extra-filename=-5641130f60a8056b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/futures-task-6662546d276b2bb5/build-script-build`
   Compiling futures-util v0.3.28
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.28/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="alloc"' --cfg 'feature="async-await"' --cfg 'feature="async-await-macro"' --cfg 'feature="channel"' --cfg 'feature="default"' --cfg 'feature="futures-channel"' --cfg 'feature="futures-io"' --cfg 'feature="futures-macro"' --cfg 'feature="futures-sink"' --cfg 'feature="io"' --cfg 'feature="memchr"' --cfg 'feature="sink"' --cfg 'feature="slab"' --cfg 'feature="std"' -C metadata=5ffe8b784c1e5ae9 -C extra-filename=-5ffe8b784c1e5ae9 --out-dir /work/oxidecomputer/crucible/target/debug/build/futures-util-5ffe8b784c1e5ae9 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name futures_task --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-task-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="std"' -C metadata=12b58f257ddc96a4 -C extra-filename=-12b58f257ddc96a4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling indexmap v1.9.3
     Running `rustc --crate-name build_script_build --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/indexmap-1.9.3/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="serde"' --cfg 'feature="serde-1"' --cfg 'feature="std"' -C metadata=69a0dfc84d6dbf8d -C extra-filename=-69a0dfc84d6dbf8d --out-dir /work/oxidecomputer/crucible/target/debug/build/indexmap-69a0dfc84d6dbf8d -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
   Compiling futures-io v0.3.28
     Running `rustc --crate-name futures_io --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-io-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=bcbbef0c8c581d67 -C extra-filename=-bcbbef0c8c581d67 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/futures-channel-224ac16fb4ca2e60/build-script-build`
     Running `rustc --crate-name futures_channel --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-channel-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="futures-sink"' --cfg 'feature="sink"' --cfg 'feature="std"' -C metadata=34a7a018f58dc5a7 -C extra-filename=-34a7a018f58dc5a7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_sink=/work/oxidecomputer/crucible/target/debug/deps/libfutures_sink-a6d6ed8a846c5f8a.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/futures-util-5ffe8b784c1e5ae9/build-script-build`
   Compiling percent-encoding v2.3.0
     Running `rustc --crate-name percent_encoding --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/percent-encoding-2.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=0000aebce3d30803 -C extra-filename=-0000aebce3d30803 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling slab v0.4.5
     Running `rustc --crate-name slab --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slab-0.4.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=5b7c79e345d6363e -C extra-filename=-5b7c79e345d6363e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/indexmap-69a0dfc84d6dbf8d/build-script-build`
   Compiling socket2 v0.5.3
     Running `rustc --crate-name socket2 --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/socket2-0.5.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="all"' -C metadata=b464b617227db85e -C extra-filename=-b464b617227db85e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling time v0.1.44
     Running `rustc --crate-name time /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/time-0.1.44/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fcbf6ea26d482f3a -C extra-filename=-fcbf6ea26d482f3a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling iana-time-zone v0.1.47
     Running `rustc --crate-name iana_time_zone --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/iana-time-zone-0.1.47/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="fallback"' -C metadata=3051bc3a975f54c9 -C extra-filename=-3051bc3a975f54c9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling tinyvec_macros v0.1.0
     Running `rustc --crate-name tinyvec_macros --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tinyvec_macros-0.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=71472bb1a287b427 -C extra-filename=-71472bb1a287b427 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling tinyvec v1.5.1
     Running `rustc --crate-name tinyvec --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tinyvec-1.5.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="tinyvec_macros"' -C metadata=592495f429dcf8bd -C extra-filename=-592495f429dcf8bd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern tinyvec_macros=/work/oxidecomputer/crucible/target/debug/deps/libtinyvec_macros-71472bb1a287b427.rmeta --cap-lints allow`
   Compiling unicode-bidi v0.3.7
     Running `rustc --crate-name unicode_bidi --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-bidi-0.3.7/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=2c825f5beb05a037 -C extra-filename=-2c825f5beb05a037 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling tracing-core v0.1.30
     Running `rustc --crate-name tracing_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-core-0.1.30/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="once_cell"' --cfg 'feature="std"' --cfg 'feature="valuable"' -C metadata=adac59f754126e83 -C extra-filename=-adac59f754126e83 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --cap-lints allow`
   Compiling schemars v0.8.12
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/schemars-0.8.12/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="bytes"' --cfg 'feature="chrono"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="schemars_derive"' --cfg 'feature="uuid1"' -C metadata=cf4f920476471968 -C extra-filename=-cf4f920476471968 --out-dir /work/oxidecomputer/crucible/target/debug/build/schemars-cf4f920476471968 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling form_urlencoded v1.2.0
     Running `rustc --crate-name form_urlencoded --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/form_urlencoded-1.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=ef731295a29c9150 -C extra-filename=-ef731295a29c9150 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --cap-lints allow`
   Compiling ring v0.16.20
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ring-0.16.20/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="dev_urandom_fallback"' --cfg 'feature="once_cell"' -C metadata=9674c8d76dda18b5 -C extra-filename=-9674c8d76dda18b5 --out-dir /work/oxidecomputer/crucible/target/debug/build/ring-9674c8d76dda18b5 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cc=/work/oxidecomputer/crucible/target/debug/deps/libcc-2976d4b8f46fa671.rlib --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/schemars-cf4f920476471968/build-script-build`
   Compiling dyn-clone v1.0.5
     Running `rustc --crate-name dyn_clone --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dyn-clone-1.0.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=6ca330b50294836a -C extra-filename=-6ca330b50294836a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling matches v0.1.9
     Running `rustc --crate-name matches /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/matches-0.1.9/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=7dd63e8ea065bea3 -C extra-filename=-7dd63e8ea065bea3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling spin v0.5.2
     Running `rustc --crate-name spin /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/spin-0.5.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bfb6115ad3135235 -C extra-filename=-bfb6115ad3135235 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling untrusted v0.7.1
     Running `rustc --crate-name untrusted --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/untrusted-0.7.1/src/untrusted.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4b93784238d33e58 -C extra-filename=-4b93784238d33e58 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling base64 v0.21.3
     Running `rustc --crate-name base64 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/base64-0.21.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=c139bdd129e780ac -C extra-filename=-c139bdd129e780ac --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling lazy_static v1.4.0
     Running `rustc --crate-name lazy_static /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/lazy_static-1.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=9c1a21fa7aad0259 -C extra-filename=-9c1a21fa7aad0259 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling rustls v0.21.6
     Running `rustc --crate-name build_script_build --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-0.21.6/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="dangerous_configuration"' --cfg 'feature="default"' --cfg 'feature="log"' --cfg 'feature="logging"' --cfg 'feature="tls12"' -C metadata=0c105edc866f624d -C extra-filename=-0c105edc866f624d --out-dir /work/oxidecomputer/crucible/target/debug/build/rustls-0c105edc866f624d -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling unicode-normalization v0.1.19
     Running `rustc --crate-name unicode_normalization --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-normalization-0.1.19/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=3df8261a03d4248e -C extra-filename=-3df8261a03d4248e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern tinyvec=/work/oxidecomputer/crucible/target/debug/deps/libtinyvec-592495f429dcf8bd.rmeta --cap-lints allow`
   Compiling heck v0.4.1
     Running `rustc --crate-name heck --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/heck-0.4.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=29af75c938b110f7 -C extra-filename=-29af75c938b110f7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling ppv-lite86 v0.2.16
     Running `rustc --crate-name ppv_lite86 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ppv-lite86-0.2.16/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="simd"' --cfg 'feature="std"' -C metadata=a7fd0e8d54744928 -C extra-filename=-a7fd0e8d54744928 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling anyhow v1.0.71
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.71/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=6798eab25f59189e -C extra-filename=-6798eab25f59189e --out-dir /work/oxidecomputer/crucible/target/debug/build/anyhow-6798eab25f59189e -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/ring-9674c8d76dda18b5/build-script-build`
   Compiling rustls-pemfile v1.0.3
     Running `rustc --crate-name rustls_pemfile --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-pemfile-1.0.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e52b2a6debfcae48 -C extra-filename=-e52b2a6debfcae48 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --cap-lints allow`
   Compiling openssl-sys v0.9.90
     Running `rustc --crate-name build_script_main --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-sys-0.9.90/build/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=b13a834c02d58f09 -C extra-filename=-b13a834c02d58f09 --out-dir /work/oxidecomputer/crucible/target/debug/build/openssl-sys-b13a834c02d58f09 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cc=/work/oxidecomputer/crucible/target/debug/deps/libcc-2976d4b8f46fa671.rlib --extern pkg_config=/work/oxidecomputer/crucible/target/debug/deps/libpkg_config-e05c47386f4bdcc0.rlib --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/anyhow-6798eab25f59189e/build-script-build`
   Compiling rand_chacha v0.3.1
     Running `rustc --crate-name rand_chacha --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_chacha-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=add466c063ef8725 -C extra-filename=-add466c063ef8725 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ppv_lite86=/work/oxidecomputer/crucible/target/debug/deps/libppv_lite86-a7fd0e8d54744928.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-e2870cc0e01c33c9.rmeta --cap-lints allow`
     Running `rustc --crate-name anyhow --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.71/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=c23d7ea2a714731e -C extra-filename=-c23d7ea2a714731e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling httparse v1.8.0
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/httparse-1.8.0/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=0ca1fa0577ea1bb6 -C extra-filename=-0ca1fa0577ea1bb6 --out-dir /work/oxidecomputer/crucible/target/debug/build/httparse-0ca1fa0577ea1bb6 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling serde_derive_internals v0.26.0
   Compiling idna v0.2.3
     Running `rustc --crate-name serde_derive_internals /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_derive_internals-0.26.0/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=6aea929a774cf30f -C extra-filename=-6aea929a774cf30f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --cap-lints allow`
     Running `rustc --crate-name idna --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/idna-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=67f12269d91917c9 -C extra-filename=-67f12269d91917c9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern matches=/work/oxidecomputer/crucible/target/debug/deps/libmatches-7dd63e8ea065bea3.rmeta --extern unicode_bidi=/work/oxidecomputer/crucible/target/debug/deps/libunicode_bidi-2c825f5beb05a037.rmeta --extern unicode_normalization=/work/oxidecomputer/crucible/target/debug/deps/libunicode_normalization-3df8261a03d4248e.rmeta --cap-lints allow`
   Compiling rand v0.8.5
     Running `rustc --crate-name rand --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand-0.8.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="getrandom"' --cfg 'feature="libc"' --cfg 'feature="min_const_gen"' --cfg 'feature="rand_chacha"' --cfg 'feature="small_rng"' --cfg 'feature="std"' --cfg 'feature="std_rng"' -C metadata=1f91a9ea4aed49ee -C extra-filename=-1f91a9ea4aed49ee --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-e2870cc0e01c33c9.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/openssl-sys-b13a834c02d58f09/build-script-main`
     Running `/work/oxidecomputer/crucible/target/debug/build/httparse-0ca1fa0577ea1bb6/build-script-build`
   Compiling socket2 v0.4.9
     Running `rustc --crate-name socket2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/socket2-0.4.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="all"' -C metadata=3c3e3607c1c6d64e -C extra-filename=-3c3e3607c1c6d64e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling try-lock v0.2.3
     Running `rustc --crate-name try_lock /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/try-lock-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=02baba71598f88d3 -C extra-filename=-02baba71598f88d3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling want v0.3.0
     Running `rustc --crate-name want --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/want-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5737a0d118420ef7 -C extra-filename=-5737a0d118420ef7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern try_lock=/work/oxidecomputer/crucible/target/debug/deps/libtry_lock-02baba71598f88d3.rmeta --cap-lints allow`
     Running `rustc --crate-name httparse --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/httparse-1.8.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=59406412a39ce707 -C extra-filename=-59406412a39ce707 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg httparse_simd`
   Compiling url v2.2.2
     Running `rustc --crate-name url --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/url-2.2.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ff56943ab9066fdc -C extra-filename=-ff56943ab9066fdc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern form_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libform_urlencoded-ef731295a29c9150.rmeta --extern idna=/work/oxidecomputer/crucible/target/debug/deps/libidna-67f12269d91917c9.rmeta --extern matches=/work/oxidecomputer/crucible/target/debug/deps/libmatches-7dd63e8ea065bea3.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --cap-lints allow`
   Compiling tower-service v0.3.1
     Running `rustc --crate-name tower_service --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tower-service-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=51da71f2ad5117ee -C extra-filename=-51da71f2ad5117ee --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling httpdate v1.0.2
     Running `rustc --crate-name httpdate --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/httpdate-1.0.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e3ef82a990113a54 -C extra-filename=-e3ef82a990113a54 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling ahash v0.8.3
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ahash-0.8.3/./build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=a145e18de00de996 -C extra-filename=-a145e18de00de996 --out-dir /work/oxidecomputer/crucible/target/debug/build/ahash-a145e18de00de996 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern version_check=/work/oxidecomputer/crucible/target/debug/deps/libversion_check-23142ed5bf6178a0.rlib --cap-lints allow`
   Compiling typenum v1.15.0
     Running `rustc --crate-name build_script_main --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/typenum-1.15.0/build/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=bfc33f8cedea7839 -C extra-filename=-bfc33f8cedea7839 --out-dir /work/oxidecomputer/crucible/target/debug/build/typenum-bfc33f8cedea7839 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling crossbeam-utils v0.8.8
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossbeam-utils-0.8.8/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="lazy_static"' --cfg 'feature="std"' -C metadata=bb0b9e93c375ad11 -C extra-filename=-bb0b9e93c375ad11 --out-dir /work/oxidecomputer/crucible/target/debug/build/crossbeam-utils-bb0b9e93c375ad11 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling foreign-types-shared v0.1.1
     Running `rustc --crate-name foreign_types_shared /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/foreign-types-shared-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=623b4e0ee39a8057 -C extra-filename=-623b4e0ee39a8057 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling openssl v0.10.55
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-0.10.55/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' -C metadata=754c33330368d7dd -C extra-filename=-754c33330368d7dd --out-dir /work/oxidecomputer/crucible/target/debug/build/openssl-754c33330368d7dd -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling foreign-types v0.3.2
     Running `rustc --crate-name foreign_types /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/foreign-types-0.3.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=07949a1c938aca9c -C extra-filename=-07949a1c938aca9c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern foreign_types_shared=/work/oxidecomputer/crucible/target/debug/deps/libforeign_types_shared-623b4e0ee39a8057.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/ahash-a145e18de00de996/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/crossbeam-utils-bb0b9e93c375ad11/build-script-build`
     Running `rustc --crate-name openssl_sys --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-sys-0.9.90/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fe2ced2b5ac7bf5a -C extra-filename=-fe2ced2b5ac7bf5a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow -L native=/usr/ssl-3/lib/amd64 -l ssl -l crypto --cfg openssl --cfg 'osslconf="OPENSSL_NO_SSL3_METHOD"' --cfg ossl300 --cfg ossl101 --cfg ossl102 --cfg ossl102f --cfg ossl102h --cfg ossl110 --cfg ossl110f --cfg ossl110g --cfg ossl110h --cfg ossl111 --cfg ossl111b --cfg ossl111c --cfg ossl111d`
   Compiling generic-array v0.14.5
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generic-array-0.14.5/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="more_lengths"' -C metadata=3cb0d81c60829d14 -C extra-filename=-3cb0d81c60829d14 --out-dir /work/oxidecomputer/crucible/target/debug/build/generic-array-3cb0d81c60829d14 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern version_check=/work/oxidecomputer/crucible/target/debug/deps/libversion_check-23142ed5bf6178a0.rlib --cap-lints allow`
   Compiling native-tls v0.2.11
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/native-tls-0.2.11/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=52d6aeab7008e848 -C extra-filename=-52d6aeab7008e848 --out-dir /work/oxidecomputer/crucible/target/debug/build/native-tls-52d6aeab7008e848 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/openssl-754c33330368d7dd/build-script-build`
     Running `rustc --crate-name ahash --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ahash-0.8.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0c4a58ad1daaf354 -C extra-filename=-0c4a58ad1daaf354 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --cap-lints allow --cfg 'feature="folded_multiply"'`
     Running `/work/oxidecomputer/crucible/target/debug/build/typenum-bfc33f8cedea7839/build-script-main`
     Running `rustc --crate-name typenum --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/typenum-1.15.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=21543e9c4f362850 -C extra-filename=-21543e9c4f362850 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/generic-array-3cb0d81c60829d14/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/native-tls-52d6aeab7008e848/build-script-build`
     Running `rustc --crate-name crossbeam_utils --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossbeam-utils-0.8.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="lazy_static"' --cfg 'feature="std"' -C metadata=a1fb255bfa31483a -C extra-filename=-a1fb255bfa31483a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --cap-lints allow`
   Compiling openssl-probe v0.1.5
     Running `rustc --crate-name openssl_probe /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-probe-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ef939aeef8872804 -C extra-filename=-ef939aeef8872804 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling unicode-width v0.1.9
     Running `rustc --crate-name unicode_width /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-width-0.1.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=87c5262a4c4bb0e9 -C extra-filename=-87c5262a4c4bb0e9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling hashbrown v0.13.2
     Running `rustc --crate-name hashbrown --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hashbrown-0.13.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="ahash"' --cfg 'feature="default"' --cfg 'feature="inline-more"' -C metadata=d4fb045aef0e24c1 -C extra-filename=-d4fb045aef0e24c1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ahash=/work/oxidecomputer/crucible/target/debug/deps/libahash-0c4a58ad1daaf354.rmeta --cap-lints allow`
   Compiling aho-corasick v1.0.1
     Running `rustc --crate-name aho_corasick --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/aho-corasick-1.0.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="perf-literal"' --cfg 'feature="std"' -C metadata=afc99e972f7e39a1 -C extra-filename=-afc99e972f7e39a1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --cap-lints allow`
   Compiling regex-syntax v0.7.2
     Running `rustc --crate-name regex_syntax --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/regex-syntax-0.7.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' --cfg 'feature="unicode"' --cfg 'feature="unicode-age"' --cfg 'feature="unicode-bool"' --cfg 'feature="unicode-case"' --cfg 'feature="unicode-gencat"' --cfg 'feature="unicode-perl"' --cfg 'feature="unicode-script"' --cfg 'feature="unicode-segment"' -C metadata=c9e6cb4ff3b1ed69 -C extra-filename=-c9e6cb4ff3b1ed69 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling mime v0.3.16
     Running `rustc --crate-name mime /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mime-0.3.16/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ac14a9115eddd3c2 -C extra-filename=-ac14a9115eddd3c2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name generic_array /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generic-array-0.14.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="more_lengths"' -C metadata=bc31b01978a602e7 -C extra-filename=-bc31b01978a602e7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern typenum=/work/oxidecomputer/crucible/target/debug/deps/libtypenum-21543e9c4f362850.rmeta --cap-lints allow --cfg relaxed_coherence`
   Compiling encoding_rs v0.8.30
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/encoding_rs-0.8.30/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="alloc"' --cfg 'feature="default"' -C metadata=99449b8c6d70454a -C extra-filename=-99449b8c6d70454a --out-dir /work/oxidecomputer/crucible/target/debug/build/encoding_rs-99449b8c6d70454a -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling byteorder v1.4.3
     Running `rustc --crate-name byteorder --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/byteorder-1.4.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=1306999913c8e1b3 -C extra-filename=-1306999913c8e1b3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling regress v0.6.0
     Running `rustc --crate-name regress --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/regress-0.6.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="backend-pikevm"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=10da65958da1c830 -C extra-filename=-10da65958da1c830 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern hashbrown=/work/oxidecomputer/crucible/target/debug/deps/libhashbrown-d4fb045aef0e24c1.rmeta --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/encoding_rs-99449b8c6d70454a/build-script-build`
     Running `rustc --crate-name encoding_rs --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/encoding_rs-0.8.30/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' -C metadata=3255048793b3f7a6 -C extra-filename=-3255048793b3f7a6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --cap-lints allow`
   Compiling crossbeam-channel v0.5.3
     Running `rustc --crate-name crossbeam_channel --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossbeam-channel-0.5.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="crossbeam-utils"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=19ff6fa486e51c63 -C extra-filename=-19ff6fa486e51c63 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern crossbeam_utils=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_utils-a1fb255bfa31483a.rmeta --cap-lints allow`
   Compiling ipnet v2.4.0
     Running `rustc --crate-name ipnet /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ipnet-2.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=8b250db103a32779 -C extra-filename=-8b250db103a32779 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling webpki-roots v0.25.2
     Running `rustc --crate-name webpki_roots --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/webpki-roots-0.25.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=31272bd9a7615638 -C extra-filename=-31272bd9a7615638 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling io-lifetimes v1.0.3
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/io-lifetimes-1.0.3/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="close"' --cfg 'feature="libc"' --cfg 'feature="windows-sys"' -C metadata=e14a4ad36d121892 -C extra-filename=-e14a4ad36d121892 --out-dir /work/oxidecomputer/crucible/target/debug/build/io-lifetimes-e14a4ad36d121892 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/io-lifetimes-e14a4ad36d121892/build-script-build`
   Compiling regex v1.8.3
     Running `rustc --crate-name regex --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/regex-1.8.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="aho-corasick"' --cfg 'feature="default"' --cfg 'feature="memchr"' --cfg 'feature="perf"' --cfg 'feature="perf-cache"' --cfg 'feature="perf-dfa"' --cfg 'feature="perf-inline"' --cfg 'feature="perf-literal"' --cfg 'feature="std"' --cfg 'feature="unicode"' --cfg 'feature="unicode-age"' --cfg 'feature="unicode-bool"' --cfg 'feature="unicode-case"' --cfg 'feature="unicode-gencat"' --cfg 'feature="unicode-perl"' --cfg 'feature="unicode-script"' --cfg 'feature="unicode-segment"' -C metadata=f9e3a4eb3da387ce -C extra-filename=-f9e3a4eb3da387ce --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern aho_corasick=/work/oxidecomputer/crucible/target/debug/deps/libaho_corasick-afc99e972f7e39a1.rmeta --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --extern regex_syntax=/work/oxidecomputer/crucible/target/debug/deps/libregex_syntax-c9e6cb4ff3b1ed69.rmeta --cap-lints allow`
   Compiling crypto-common v0.1.6
     Running `rustc --crate-name crypto_common --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crypto-common-0.1.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="getrandom"' --cfg 'feature="rand_core"' --cfg 'feature="std"' -C metadata=0953bfc5dcef84b9 -C extra-filename=-0953bfc5dcef84b9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern generic_array=/work/oxidecomputer/crucible/target/debug/deps/libgeneric_array-bc31b01978a602e7.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-e2870cc0e01c33c9.rmeta --extern typenum=/work/oxidecomputer/crucible/target/debug/deps/libtypenum-21543e9c4f362850.rmeta --cap-lints allow`
     Running `rustc --crate-name io_lifetimes --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/io-lifetimes-1.0.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="close"' --cfg 'feature="libc"' --cfg 'feature="windows-sys"' -C metadata=df7ee936a2a5cbac -C extra-filename=-df7ee936a2a5cbac --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow --cfg io_safety_is_in_std --cfg panic_in_const_fn`
   Compiling getopts v0.2.21
     Running `rustc --crate-name getopts /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/getopts-0.2.21/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=93a8419d37acce69 -C extra-filename=-93a8419d37acce69 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern unicode_width=/work/oxidecomputer/crucible/target/debug/deps/libunicode_width-87c5262a4c4bb0e9.rmeta --cap-lints allow`
   Compiling unsafe-libyaml v0.2.5
     Running `rustc --crate-name unsafe_libyaml --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unsafe-libyaml-0.2.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=18c99c193ab0b0f5 -C extra-filename=-18c99c193ab0b0f5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling serde_derive v1.0.167
     Running `rustc --crate-name serde_derive /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_derive-1.0.167/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="default"' -C metadata=3441a1f9756a6d5b -C extra-filename=-3441a1f9756a6d5b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling thiserror-impl v1.0.40
     Running `rustc --crate-name thiserror_impl --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thiserror-impl-1.0.40/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=bf767c319ff2d238 -C extra-filename=-bf767c319ff2d238 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling futures-macro v0.3.28
     Running `rustc --crate-name futures_macro --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-macro-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=81cbf12cf17c7f91 -C extra-filename=-81cbf12cf17c7f91 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling tokio-macros v2.1.0
     Running `rustc --crate-name tokio_macros --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-macros-2.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=129d27199cb07668 -C extra-filename=-129d27199cb07668 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling schemars_derive v0.8.12
     Running `rustc --crate-name schemars_derive --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/schemars_derive-0.8.12/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=ca659e99c5534de4 -C extra-filename=-ca659e99c5534de4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern serde_derive_internals=/work/oxidecomputer/crucible/target/debug/deps/libserde_derive_internals-6aea929a774cf30f.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern proc_macro --cap-lints allow`
   Compiling tracing-attributes v0.1.23
     Running `rustc --crate-name tracing_attributes --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-attributes-0.1.23/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=af60b1f7cb0d953c -C extra-filename=-af60b1f7cb0d953c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern proc_macro --cap-lints allow`
   Compiling openssl-macros v0.1.0
     Running `rustc --crate-name openssl_macros --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-macros-0.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=57d0261583d18db5 -C extra-filename=-57d0261583d18db5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern proc_macro --cap-lints allow`
   Compiling ucd-trie v0.1.3
     Running `rustc --crate-name ucd_trie --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ucd-trie-0.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=a4e8188b5963a3f1 -C extra-filename=-a4e8188b5963a3f1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name futures_util --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="async-await"' --cfg 'feature="async-await-macro"' --cfg 'feature="channel"' --cfg 'feature="default"' --cfg 'feature="futures-channel"' --cfg 'feature="futures-io"' --cfg 'feature="futures-macro"' --cfg 'feature="futures-sink"' --cfg 'feature="io"' --cfg 'feature="memchr"' --cfg 'feature="sink"' --cfg 'feature="slab"' --cfg 'feature="std"' -C metadata=b4da5d5433271d56 -C extra-filename=-b4da5d5433271d56 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_io=/work/oxidecomputer/crucible/target/debug/deps/libfutures_io-bcbbef0c8c581d67.rmeta --extern futures_macro=/work/oxidecomputer/crucible/target/debug/deps/libfutures_macro-81cbf12cf17c7f91.so --extern futures_sink=/work/oxidecomputer/crucible/target/debug/deps/libfutures_sink-a6d6ed8a846c5f8a.rmeta --extern futures_task=/work/oxidecomputer/crucible/target/debug/deps/libfutures_task-12b58f257ddc96a4.rmeta --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern pin_utils=/work/oxidecomputer/crucible/target/debug/deps/libpin_utils-bcfb754cd1ab8c67.rmeta --extern slab=/work/oxidecomputer/crucible/target/debug/deps/libslab-5b7c79e345d6363e.rmeta --cap-lints allow`
     Running `rustc --crate-name thiserror --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thiserror-1.0.40/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=07eca56f531a0e5d -C extra-filename=-07eca56f531a0e5d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern thiserror_impl=/work/oxidecomputer/crucible/target/debug/deps/libthiserror_impl-bf767c319ff2d238.so --cap-lints allow`
     Running `rustc --crate-name openssl --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openssl-0.10.55/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=4a7553c915defdd5 -C extra-filename=-4a7553c915defdd5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern foreign_types=/work/oxidecomputer/crucible/target/debug/deps/libforeign_types-07949a1c938aca9c.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern openssl_macros=/work/oxidecomputer/crucible/target/debug/deps/libopenssl_macros-57d0261583d18db5.so --extern ffi=/work/oxidecomputer/crucible/target/debug/deps/libopenssl_sys-fe2ced2b5ac7bf5a.rmeta --cap-lints allow -L native=/usr/ssl-3/lib/amd64 --cfg 'osslconf="OPENSSL_NO_SSL3_METHOD"' --cfg ossl101 --cfg ossl102 --cfg ossl110 --cfg ossl110g --cfg ossl110h --cfg ossl111 --cfg ossl300`
   Compiling pest v2.1.3
     Running `rustc --crate-name pest /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pest-2.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e1aabf8a6c7bc1ff -C extra-filename=-e1aabf8a6c7bc1ff --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ucd_trie=/work/oxidecomputer/crucible/target/debug/deps/libucd_trie-a4e8188b5963a3f1.rmeta --cap-lints allow`
   Compiling unicode-xid v0.2.2
     Running `rustc --crate-name unicode_xid /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-xid-0.2.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="default"' -C metadata=911d92403f8fb404 -C extra-filename=-911d92403f8fb404 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling maplit v1.0.2
     Running `rustc --crate-name maplit /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/maplit-1.0.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=695e3e42e8316fa9 -C extra-filename=-695e3e42e8316fa9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling synstructure v0.12.6
     Running `rustc --crate-name synstructure --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/synstructure-0.12.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="proc-macro"' -C metadata=48a8bcebf08faced -C extra-filename=-48a8bcebf08faced --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --extern unicode_xid=/work/oxidecomputer/crucible/target/debug/deps/libunicode_xid-911d92403f8fb404.rmeta --cap-lints allow`
   Compiling slog v2.7.0
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-2.7.0/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="dynamic-keys"' --cfg 'feature="max_level_trace"' --cfg 'feature="release_max_level_debug"' --cfg 'feature="std"' -C metadata=125bc9f2670a64c3 -C extra-filename=-125bc9f2670a64c3 --out-dir /work/oxidecomputer/crucible/target/debug/build/slog-125bc9f2670a64c3 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling pest_meta v2.1.3
     Running `rustc --crate-name pest_meta /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pest_meta-2.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=74e8955a7d734149 -C extra-filename=-74e8955a7d734149 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern maplit=/work/oxidecomputer/crucible/target/debug/deps/libmaplit-695e3e42e8316fa9.rmeta --extern pest=/work/oxidecomputer/crucible/target/debug/deps/libpest-e1aabf8a6c7bc1ff.rmeta --cap-lints allow`
   Compiling time-core v0.1.1
     Running `rustc --crate-name time_core --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/time-core-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fd216cb66fe61cb9 -C extra-filename=-fd216cb66fe61cb9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling subtle v2.4.1
     Running `rustc --crate-name subtle /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/subtle-2.4.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ee0c8d46ce57336b -C extra-filename=-ee0c8d46ce57336b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling tracing v0.1.37
     Running `rustc --crate-name tracing --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="attributes"' --cfg 'feature="default"' --cfg 'feature="std"' --cfg 'feature="tracing-attributes"' -C metadata=1de351a7f2b0ab55 -C extra-filename=-1de351a7f2b0ab55 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern tracing_attributes=/work/oxidecomputer/crucible/target/debug/deps/libtracing_attributes-af60b1f7cb0d953c.so --extern tracing_core=/work/oxidecomputer/crucible/target/debug/deps/libtracing_core-adac59f754126e83.rmeta --cap-lints allow`
   Compiling rustversion v1.0.14
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustversion-1.0.14/build/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=417212d646cdbd4e -C extra-filename=-417212d646cdbd4e --out-dir /work/oxidecomputer/crucible/target/debug/build/rustversion-417212d646cdbd4e -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling equivalent v1.0.0
     Running `rustc --crate-name equivalent /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/equivalent-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=7a40718821784752 -C extra-filename=-7a40718821784752 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling hashbrown v0.14.0
     Running `rustc --crate-name hashbrown --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hashbrown-0.14.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="raw"' -C metadata=615db2aaa1e4d335 -C extra-filename=-615db2aaa1e4d335 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling zerocopy-derive v0.2.0
     Running `rustc --crate-name zerocopy_derive --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zerocopy-derive-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=f3f98c5abdba899d -C extra-filename=-f3f98c5abdba899d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern synstructure=/work/oxidecomputer/crucible/target/debug/deps/libsynstructure-48a8bcebf08faced.rlib --extern proc_macro --cap-lints allow`
   Compiling time-macros v0.2.10
     Running `rustc --crate-name time_macros --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/time-macros-0.2.10/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="formatting"' --cfg 'feature="parsing"' -C metadata=e07155b8a4270ad7 -C extra-filename=-e07155b8a4270ad7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern time_core=/work/oxidecomputer/crucible/target/debug/deps/libtime_core-fd216cb66fe61cb9.rlib --extern proc_macro --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/rustversion-417212d646cdbd4e/build-script-build`
   Compiling pest_generator v2.1.3
     Running `rustc --crate-name pest_generator /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pest_generator-2.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=01755500ddb0705d -C extra-filename=-01755500ddb0705d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern pest=/work/oxidecomputer/crucible/target/debug/deps/libpest-e1aabf8a6c7bc1ff.rmeta --extern pest_meta=/work/oxidecomputer/crucible/target/debug/deps/libpest_meta-74e8955a7d734149.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --cap-lints allow`
   Compiling indexmap v2.0.0
     Running `rustc --crate-name indexmap --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/indexmap-2.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=32a62b7926f710bd -C extra-filename=-32a62b7926f710bd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern equivalent=/work/oxidecomputer/crucible/target/debug/deps/libequivalent-7a40718821784752.rmeta --extern hashbrown=/work/oxidecomputer/crucible/target/debug/deps/libhashbrown-615db2aaa1e4d335.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/slog-125bc9f2670a64c3/build-script-build`
   Compiling num_threads v0.1.5
     Running `rustc --crate-name num_threads /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num_threads-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4e84d104f1db9110 -C extra-filename=-4e84d104f1db9110 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling winnow v0.4.6
     Running `rustc --crate-name winnow --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/winnow-0.4.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=e34c187c773d92ef -C extra-filename=-e34c187c773d92ef --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling rustix v0.37.7
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustix-0.37.7/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="fs"' --cfg 'feature="io-lifetimes"' --cfg 'feature="libc"' --cfg 'feature="std"' --cfg 'feature="use-libc-auxv"' -C metadata=b6284316c7951b5c -C extra-filename=-b6284316c7951b5c --out-dir /work/oxidecomputer/crucible/target/debug/build/rustix-b6284316c7951b5c -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name slog /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-2.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="dynamic-keys"' --cfg 'feature="max_level_trace"' --cfg 'feature="release_max_level_debug"' --cfg 'feature="std"' -C metadata=84fd25666c3c26ee -C extra-filename=-84fd25666c3c26ee --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg integer128`
     Running `rustc --crate-name serde /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde-1.0.167/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="rc"' --cfg 'feature="serde_derive"' --cfg 'feature="std"' -C metadata=2779165e31567af2 -C extra-filename=-2779165e31567af2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde_derive=/work/oxidecomputer/crucible/target/debug/deps/libserde_derive-3441a1f9756a6d5b.so --cap-lints allow`
   Compiling pest_derive v2.1.0
     Running `rustc --crate-name pest_derive /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pest_derive-2.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=8b18b291ae7a4e87 -C extra-filename=-8b18b291ae7a4e87 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern pest=/work/oxidecomputer/crucible/target/debug/deps/libpest-e1aabf8a6c7bc1ff.rlib --extern pest_generator=/work/oxidecomputer/crucible/target/debug/deps/libpest_generator-01755500ddb0705d.rlib --extern proc_macro --cap-lints allow`
   Compiling futures-executor v0.3.28
     Running `rustc --crate-name futures_executor --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-executor-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=61f53162107ffb32 -C extra-filename=-61f53162107ffb32 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_task=/work/oxidecomputer/crucible/target/debug/deps/libfutures_task-12b58f257ddc96a4.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --cap-lints allow`
     Running `rustc --crate-name native_tls /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/native-tls-0.2.11/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=320c05ab5bbd33c9 -C extra-filename=-320c05ab5bbd33c9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern openssl=/work/oxidecomputer/crucible/target/debug/deps/libopenssl-4a7553c915defdd5.rmeta --extern openssl_probe=/work/oxidecomputer/crucible/target/debug/deps/libopenssl_probe-ef939aeef8872804.rmeta --extern openssl_sys=/work/oxidecomputer/crucible/target/debug/deps/libopenssl_sys-fe2ced2b5ac7bf5a.rmeta --cap-lints allow -L native=/usr/ssl-3/lib/amd64 --cfg have_min_max_version`
   Compiling time v0.3.23
     Running `rustc --crate-name time --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/time-0.3.23/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="formatting"' --cfg 'feature="local-offset"' --cfg 'feature="macros"' --cfg 'feature="parsing"' --cfg 'feature="std"' -C metadata=9b604407a0d52f86 -C extra-filename=-9b604407a0d52f86 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern num_threads=/work/oxidecomputer/crucible/target/debug/deps/libnum_threads-4e84d104f1db9110.rmeta --extern time_core=/work/oxidecomputer/crucible/target/debug/deps/libtime_core-fd216cb66fe61cb9.rmeta --extern time_macros=/work/oxidecomputer/crucible/target/debug/deps/libtime_macros-e07155b8a4270ad7.so --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/rustix-b6284316c7951b5c/build-script-build`
   Compiling zerocopy v0.3.0
     Running `rustc --crate-name zerocopy --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zerocopy-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0b893e01c09e6e03 -C extra-filename=-0b893e01c09e6e03 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern zerocopy_derive=/work/oxidecomputer/crucible/target/debug/deps/libzerocopy_derive-f3f98c5abdba899d.so --cap-lints allow`
     Running `rustc --crate-name rustversion --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustversion-1.0.14/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=ae2ca065d7ce77a2 -C extra-filename=-ae2ca065d7ce77a2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro --cap-lints allow`
   Compiling errno v0.3.1
     Running `rustc --crate-name errno --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/errno-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e243723ea5172a32 -C extra-filename=-e243723ea5172a32 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling usdt-impl v0.3.5
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/usdt-impl-0.3.5/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="asm"' -C metadata=d6a2bc3071c67fed -C extra-filename=-d6a2bc3071c67fed --out-dir /work/oxidecomputer/crucible/target/debug/build/usdt-impl-d6a2bc3071c67fed -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern version_check=/work/oxidecomputer/crucible/target/debug/deps/libversion_check-23142ed5bf6178a0.rlib --cap-lints allow`
   Compiling semver v1.0.18
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/semver-1.0.18/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="std"' -C metadata=6244f63ae51a7b8f -C extra-filename=-6244f63ae51a7b8f --out-dir /work/oxidecomputer/crucible/target/debug/build/semver-6244f63ae51a7b8f -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling async-trait v0.1.73
     Running `rustc --crate-name build_script_build --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-trait-0.1.73/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=72eabf91258cb242 -C extra-filename=-72eabf91258cb242 --out-dir /work/oxidecomputer/crucible/target/debug/build/async-trait-72eabf91258cb242 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/usdt-impl-d6a2bc3071c67fed/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/semver-6244f63ae51a7b8f/build-script-build`
   Compiling dof v0.1.5
     Running `rustc --crate-name dof --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dof-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=48f06b38719b0295 -C extra-filename=-48f06b38719b0295 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern zerocopy=/work/oxidecomputer/crucible/target/debug/deps/libzerocopy-0b893e01c09e6e03.rmeta --cap-lints allow`
     Running `rustc --crate-name rustix --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustix-0.37.7/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="fs"' --cfg 'feature="io-lifetimes"' --cfg 'feature="libc"' --cfg 'feature="std"' --cfg 'feature="use-libc-auxv"' -C metadata=4f0213bb214bbfd6 -C extra-filename=-4f0213bb214bbfd6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern libc_errno=/work/oxidecomputer/crucible/target/debug/deps/liberrno-e243723ea5172a32.rmeta --extern io_lifetimes=/work/oxidecomputer/crucible/target/debug/deps/libio_lifetimes-df7ee936a2a5cbac.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow --cfg libc --cfg solarish`
     Running `/work/oxidecomputer/crucible/target/debug/build/async-trait-72eabf91258cb242/build-script-build`
   Compiling dtrace-parser v0.1.14
     Running `rustc --crate-name dtrace_parser --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dtrace-parser-0.1.14/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=647a421ab06e4ff3 -C extra-filename=-647a421ab06e4ff3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern pest=/work/oxidecomputer/crucible/target/debug/deps/libpest-e1aabf8a6c7bc1ff.rmeta --extern pest_derive=/work/oxidecomputer/crucible/target/debug/deps/libpest_derive-8b18b291ae7a4e87.so --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --cap-lints allow`
   Compiling thread-id v4.0.0
     Running `rustc --crate-name thread_id /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thread-id-4.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=eaa0d9ff93152533 -C extra-filename=-eaa0d9ff93152533 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling fastrand v1.7.0
     Running `rustc --crate-name fastrand --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fastrand-1.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=d1e60a5c45d284ad -C extra-filename=-d1e60a5c45d284ad --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name async_trait --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-trait-0.1.73/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=a300e84178ee0ad1 -C extra-filename=-a300e84178ee0ad1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling block-buffer v0.10.2
     Running `rustc --crate-name block_buffer --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/block-buffer-0.10.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0de17b0eee43f62e -C extra-filename=-0de17b0eee43f62e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern generic_array=/work/oxidecomputer/crucible/target/debug/deps/libgeneric_array-bc31b01978a602e7.rmeta --cap-lints allow`
   Compiling thread_local v1.1.4
     Running `rustc --crate-name thread_local --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thread_local-1.1.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=771d112d9219dc9f -C extra-filename=-771d112d9219dc9f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --cap-lints allow`
   Compiling strsim v0.10.0
     Running `rustc --crate-name strsim /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/strsim-0.10.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=3314ed6444b5bb69 -C extra-filename=-3314ed6444b5bb69 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling digest v0.10.6
     Running `rustc --crate-name digest --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/digest-0.10.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="block-buffer"' --cfg 'feature="core-api"' --cfg 'feature="default"' --cfg 'feature="mac"' --cfg 'feature="std"' --cfg 'feature="subtle"' -C metadata=d0d34473efe105d1 -C extra-filename=-d0d34473efe105d1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern block_buffer=/work/oxidecomputer/crucible/target/debug/deps/libblock_buffer-0de17b0eee43f62e.rmeta --extern crypto_common=/work/oxidecomputer/crucible/target/debug/deps/libcrypto_common-0953bfc5dcef84b9.rmeta --extern subtle=/work/oxidecomputer/crucible/target/debug/deps/libsubtle-ee0c8d46ce57336b.rmeta --cap-lints allow`
   Compiling tempfile v3.5.0
     Running `rustc --crate-name tempfile --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tempfile-3.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=05cb6a5f7e86cb66 -C extra-filename=-05cb6a5f7e86cb66 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern fastrand=/work/oxidecomputer/crucible/target/debug/deps/libfastrand-d1e60a5c45d284ad.rmeta --extern rustix=/work/oxidecomputer/crucible/target/debug/deps/librustix-4f0213bb214bbfd6.rmeta --cap-lints allow`
   Compiling dirs-sys-next v0.1.2
     Running `rustc --crate-name dirs_sys_next --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dirs-sys-next-0.1.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4419c59e785c767e -C extra-filename=-4419c59e785c767e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling dirs-next v2.0.0
     Running `rustc --crate-name dirs_next --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dirs-next-2.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=369b0b19f2fe8c2c -C extra-filename=-369b0b19f2fe8c2c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern dirs_sys_next=/work/oxidecomputer/crucible/target/debug/deps/libdirs_sys_next-4419c59e785c767e.rmeta --cap-lints allow`
   Compiling libz-sys v1.1.8
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libz-sys-1.1.8/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="libc"' -C metadata=664a34d733156495 -C extra-filename=-664a34d733156495 --out-dir /work/oxidecomputer/crucible/target/debug/build/libz-sys-664a34d733156495 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cc=/work/oxidecomputer/crucible/target/debug/deps/libcc-2976d4b8f46fa671.rlib --extern pkg_config=/work/oxidecomputer/crucible/target/debug/deps/libpkg_config-e05c47386f4bdcc0.rlib --cap-lints allow`
   Compiling siphasher v0.3.10
     Running `rustc --crate-name siphasher --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/siphasher-0.3.10/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=2eb27360f66646f3 -C extra-filename=-2eb27360f66646f3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling match_cfg v0.1.0
     Running `rustc --crate-name match_cfg /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/match_cfg-0.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="use_core"' -C metadata=a8964d97fe20de7b -C extra-filename=-a8964d97fe20de7b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling slog-async v2.8.0
   Compiling fallible-iterator v0.2.0
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-async-2.8.0/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' -C metadata=128f43cfd1e1203b -C extra-filename=-128f43cfd1e1203b --out-dir /work/oxidecomputer/crucible/target/debug/build/slog-async-128f43cfd1e1203b -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name fallible_iterator --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fallible-iterator-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=2f475e2aff163c98 -C extra-filename=-2f475e2aff163c98 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling hostname v0.3.1
     Running `rustc --crate-name hostname /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hostname-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=4a0f8b1a56e5681a -C extra-filename=-4a0f8b1a56e5681a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern match_cfg=/work/oxidecomputer/crucible/target/debug/deps/libmatch_cfg-a8964d97fe20de7b.rmeta --cap-lints allow`
   Compiling term v0.7.0
     Running `rustc --crate-name term --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/term-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=9ecdb763f4f627bf -C extra-filename=-9ecdb763f4f627bf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dirs_next=/work/oxidecomputer/crucible/target/debug/deps/libdirs_next-369b0b19f2fe8c2c.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/libz-sys-664a34d733156495/build-script-build`
   Compiling futures v0.3.28
     Running `rustc --crate-name futures --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-0.3.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="async-await"' --cfg 'feature="default"' --cfg 'feature="executor"' --cfg 'feature="futures-executor"' --cfg 'feature="std"' -C metadata=df761c89bfa71e54 -C extra-filename=-df761c89bfa71e54 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_executor=/work/oxidecomputer/crucible/target/debug/deps/libfutures_executor-61f53162107ffb32.rmeta --extern futures_io=/work/oxidecomputer/crucible/target/debug/deps/libfutures_io-bcbbef0c8c581d67.rmeta --extern futures_sink=/work/oxidecomputer/crucible/target/debug/deps/libfutures_sink-a6d6ed8a846c5f8a.rmeta --extern futures_task=/work/oxidecomputer/crucible/target/debug/deps/libfutures_task-12b58f257ddc96a4.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --cap-lints allow`
   Compiling atty v0.2.14
     Running `rustc --crate-name atty /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/atty-0.2.14/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bfb6a2cdc762f7c4 -C extra-filename=-bfb6a2cdc762f7c4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling autocfg v0.1.8
     Running `rustc --crate-name autocfg /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/autocfg-0.1.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=0a34a0aae5d9487a -C extra-filename=-0a34a0aae5d9487a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/slog-async-128f43cfd1e1203b/build-script-build`
   Compiling cpufeatures v0.2.1
     Running `rustc --crate-name cpufeatures --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/cpufeatures-0.2.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=6e25ef20a8fa218f -C extra-filename=-6e25ef20a8fa218f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling take_mut v0.2.2
     Running `rustc --crate-name take_mut /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/take_mut-0.2.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0ebd75564ded4875 -C extra-filename=-0ebd75564ded4875 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling rand_core v0.4.2
     Running `rustc --crate-name rand_core /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_core-0.4.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="std"' -C metadata=cf5252fd1c1f594a -C extra-filename=-cf5252fd1c1f594a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling libgit2-sys v0.15.2+1.6.4
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libgit2-sys-0.15.2+1.6.4/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=69da610a4d2f950f -C extra-filename=-69da610a4d2f950f --out-dir /work/oxidecomputer/crucible/target/debug/build/libgit2-sys-69da610a4d2f950f -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cc=/work/oxidecomputer/crucible/target/debug/deps/libcc-2976d4b8f46fa671.rlib --extern pkg_config=/work/oxidecomputer/crucible/target/debug/deps/libpkg_config-e05c47386f4bdcc0.rlib --cap-lints allow`
   Compiling slog-term v2.9.0
     Running `rustc --crate-name slog_term --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-term-2.9.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=31c687431f6dd53c -C extra-filename=-31c687431f6dd53c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern atty=/work/oxidecomputer/crucible/target/debug/deps/libatty-bfb6a2cdc762f7c4.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern term=/work/oxidecomputer/crucible/target/debug/deps/libterm-9ecdb763f4f627bf.rmeta --extern thread_local=/work/oxidecomputer/crucible/target/debug/deps/libthread_local-771d112d9219dc9f.rmeta --extern time=/work/oxidecomputer/crucible/target/debug/deps/libtime-9b604407a0d52f86.rmeta --cap-lints allow`
     Running `rustc --crate-name slog_async /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-async-2.8.0/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=03e743f036119074 -C extra-filename=-03e743f036119074 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crossbeam_channel=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_channel-19ff6fa486e51c63.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern take_mut=/work/oxidecomputer/crucible/target/debug/deps/libtake_mut-0ebd75564ded4875.rmeta --extern thread_local=/work/oxidecomputer/crucible/target/debug/deps/libthread_local-771d112d9219dc9f.rmeta --cap-lints allow --cfg integer128`
   Compiling bytes v1.4.0
     Running `rustc --crate-name bytes --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bytes-1.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="std"' -C metadata=64a8a55ef81e55dd -C extra-filename=-64a8a55ef81e55dd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
     Running `rustc --crate-name serde_json --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_json-1.0.105/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=96027e7dd982d07a -C extra-filename=-96027e7dd982d07a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern ryu=/work/oxidecomputer/crucible/target/debug/deps/libryu-062e5ac4087417b3.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow --cfg limb_width_64`
     Running `rustc --crate-name indexmap --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/indexmap-1.9.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="serde"' --cfg 'feature="serde-1"' --cfg 'feature="std"' -C metadata=c2c8f74266eebb64 -C extra-filename=-c2c8f74266eebb64 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern hashbrown=/work/oxidecomputer/crucible/target/debug/deps/libhashbrown-3ad6614047c487f9.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow --cfg has_std`
   Compiling chrono v0.4.26
     Running `rustc --crate-name chrono --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/chrono-0.4.26/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="clock"' --cfg 'feature="default"' --cfg 'feature="iana-time-zone"' --cfg 'feature="js-sys"' --cfg 'feature="oldtime"' --cfg 'feature="serde"' --cfg 'feature="std"' --cfg 'feature="time"' --cfg 'feature="wasm-bindgen"' --cfg 'feature="wasmbind"' --cfg 'feature="winapi"' -C metadata=0402a5e17dd99cdf -C extra-filename=-0402a5e17dd99cdf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern iana_time_zone=/work/oxidecomputer/crucible/target/debug/deps/libiana_time_zone-3051bc3a975f54c9.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern time=/work/oxidecomputer/crucible/target/debug/deps/libtime-fcbf6ea26d482f3a.rmeta --cap-lints allow`
   Compiling uuid v1.4.1
     Running `rustc --crate-name uuid --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/uuid-1.4.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="getrandom"' --cfg 'feature="rng"' --cfg 'feature="serde"' --cfg 'feature="std"' --cfg 'feature="v4"' -C metadata=7cc8e87b4149b49e -C extra-filename=-7cc8e87b4149b49e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern getrandom=/work/oxidecomputer/crucible/target/debug/deps/libgetrandom-567199de146d617e.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling tokio v1.32.0
     Running `rustc --crate-name tokio --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.32.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bytes"' --cfg 'feature="default"' --cfg 'feature="fs"' --cfg 'feature="full"' --cfg 'feature="io-std"' --cfg 'feature="io-util"' --cfg 'feature="libc"' --cfg 'feature="macros"' --cfg 'feature="mio"' --cfg 'feature="net"' --cfg 'feature="num_cpus"' --cfg 'feature="parking_lot"' --cfg 'feature="process"' --cfg 'feature="rt"' --cfg 'feature="rt-multi-thread"' --cfg 'feature="signal"' --cfg 'feature="signal-hook-registry"' --cfg 'feature="socket2"' --cfg 'feature="sync"' --cfg 'feature="test-util"' --cfg 'feature="time"' --cfg 'feature="tokio-macros"' --cfg 'feature="windows-sys"' -C metadata=cf190744403b2ee1 -C extra-filename=-cf190744403b2ee1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern mio=/work/oxidecomputer/crucible/target/debug/deps/libmio-27a8136cf12de2bb.rmeta --extern num_cpus=/work/oxidecomputer/crucible/target/debug/deps/libnum_cpus-67a451bebfcc5086.rmeta --extern parking_lot=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot-970d5c0acece447c.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern signal_hook_registry=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_registry-4d955479f235827e.rmeta --extern socket2=/work/oxidecomputer/crucible/target/debug/deps/libsocket2-b464b617227db85e.rmeta --extern tokio_macros=/work/oxidecomputer/crucible/target/debug/deps/libtokio_macros-129d27199cb07668.so --cap-lints allow`
   Compiling http v0.2.9
     Running `rustc --crate-name http --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/http-0.2.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4526ab951518c907 -C extra-filename=-4526ab951518c907 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --cap-lints allow`
   Compiling serde_urlencoded v0.7.1
     Running `rustc --crate-name serde_urlencoded --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_urlencoded-0.7.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=779d3ac41d95557a -C extra-filename=-779d3ac41d95557a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern form_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libform_urlencoded-ef731295a29c9150.rmeta --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern ryu=/work/oxidecomputer/crucible/target/debug/deps/libryu-062e5ac4087417b3.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling serde_tokenstream v0.2.0
     Running `rustc --crate-name serde_tokenstream --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_tokenstream-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=2a83027836bfd0fb -C extra-filename=-2a83027836bfd0fb --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rmeta --cap-lints allow`
   Compiling serde_yaml v0.9.17
     Running `rustc --crate-name serde_yaml --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_yaml-0.9.17/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=c7db3f53bbf8134b -C extra-filename=-c7db3f53bbf8134b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern ryu=/work/oxidecomputer/crucible/target/debug/deps/libryu-062e5ac4087417b3.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern unsafe_libyaml=/work/oxidecomputer/crucible/target/debug/deps/libunsafe_libyaml-18c99c193ab0b0f5.rmeta --cap-lints allow`
   Compiling serde_spanned v0.6.3
     Running `rustc --crate-name serde_spanned --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_spanned-0.6.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="serde"' -C metadata=92d13640a4d5d836 -C extra-filename=-92d13640a4d5d836 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling openapiv3 v1.0.2
     Running `rustc --crate-name openapiv3 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/openapiv3-1.0.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="skip_serializing_defaults"' -C metadata=f84072cef6d0d68c -C extra-filename=-f84072cef6d0d68c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --cap-lints allow`
   Compiling toml_datetime v0.6.3
     Running `rustc --crate-name toml_datetime --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/toml_datetime-0.6.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="serde"' -C metadata=c003080b79dfe49d -C extra-filename=-c003080b79dfe49d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
     Running `rustc --crate-name usdt_impl --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/usdt-impl-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="asm"' -C metadata=aa84cb563c29877d -C extra-filename=-aa84cb563c29877d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern dof=/work/oxidecomputer/crucible/target/debug/deps/libdof-48f06b38719b0295.rmeta --extern dtrace_parser=/work/oxidecomputer/crucible/target/debug/deps/libdtrace_parser-647a421ab06e4ff3.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern thread_id=/work/oxidecomputer/crucible/target/debug/deps/libthread_id-eaa0d9ff93152533.rmeta --cap-lints allow --cfg usdt_stable_asm --cfg usdt_backend_standard`
   Compiling toml_edit v0.19.12
     Running `rustc --crate-name toml_edit --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/toml_edit-0.19.12/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="serde"' -C metadata=29ed73c573391add -C extra-filename=-29ed73c573391add --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-32a62b7926f710bd.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_spanned=/work/oxidecomputer/crucible/target/debug/deps/libserde_spanned-92d13640a4d5d836.rmeta --extern toml_datetime=/work/oxidecomputer/crucible/target/debug/deps/libtoml_datetime-c003080b79dfe49d.rmeta --extern winnow=/work/oxidecomputer/crucible/target/debug/deps/libwinnow-e34c187c773d92ef.rmeta --cap-lints allow`
     Running `rustc --crate-name schemars --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/schemars-0.8.12/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bytes"' --cfg 'feature="chrono"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="schemars_derive"' --cfg 'feature="uuid1"' -C metadata=83d20014cee5d9b5 -C extra-filename=-83d20014cee5d9b5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern dyn_clone=/work/oxidecomputer/crucible/target/debug/deps/libdyn_clone-6ca330b50294836a.rmeta --extern schemars_derive=/work/oxidecomputer/crucible/target/debug/deps/libschemars_derive-ca659e99c5534de4.so --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern uuid1=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow --cfg std_atomic64 --cfg std_atomic`
   Compiling http-body v0.4.4
     Running `rustc --crate-name http_body --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/http-body-0.4.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bb1d69dd918c127f -C extra-filename=-bb1d69dd918c127f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --cap-lints allow`
     Running `rustc --crate-name semver --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/semver-1.0.18/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="std"' -C metadata=8c1c5827befd93e7 -C extra-filename=-8c1c5827befd93e7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling serde_tokenstream v0.1.6
     Running `rustc --crate-name serde_tokenstream --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_tokenstream-0.1.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=a3f756c2f035b223 -C extra-filename=-a3f756c2f035b223 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --cap-lints allow`
   Compiling slog-json v2.6.1
     Running `rustc --crate-name slog_json --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-json-2.6.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=d8408f8f3a6dd5b7 -C extra-filename=-d8408f8f3a6dd5b7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern time=/work/oxidecomputer/crucible/target/debug/deps/libtime-9b604407a0d52f86.rmeta --cap-lints allow`
   Compiling usdt-macro v0.3.5
     Running `rustc --crate-name usdt_macro --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/usdt-macro-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="asm"' -C metadata=0ea5e72aa4bb61c5 -C extra-filename=-0ea5e72aa4bb61c5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dtrace_parser=/work/oxidecomputer/crucible/target/debug/deps/libdtrace_parser-647a421ab06e4ff3.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern serde_tokenstream=/work/oxidecomputer/crucible/target/debug/deps/libserde_tokenstream-a3f756c2f035b223.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern usdt_impl=/work/oxidecomputer/crucible/target/debug/deps/libusdt_impl-aa84cb563c29877d.rlib --extern proc_macro --cap-lints allow`
   Compiling typify-impl v0.0.13 (https://github.com/oxidecomputer/typify#92bfed8b)
     Running `rustc --crate-name typify_impl --edition=2021 /home/build/.cargo/git/checkouts/typify-288d5a84bbbe6a46/92bfed8/typify-impl/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5385296f2ea50467 -C extra-filename=-5385296f2ea50467 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern heck=/work/oxidecomputer/crucible/target/debug/deps/libheck-29af75c938b110f7.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern regress=/work/oxidecomputer/crucible/target/debug/deps/libregress-10da65958da1c830.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern unicode_ident=/work/oxidecomputer/crucible/target/debug/deps/libunicode_ident-81e7752fff89e70f.rmeta --cap-lints allow`
   Compiling usdt-attr-macro v0.3.5
     Running `rustc --crate-name usdt_attr_macro --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/usdt-attr-macro-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="asm"' -C metadata=1e49091236cd57b8 -C extra-filename=-1e49091236cd57b8 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dtrace_parser=/work/oxidecomputer/crucible/target/debug/deps/libdtrace_parser-647a421ab06e4ff3.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern serde_tokenstream=/work/oxidecomputer/crucible/target/debug/deps/libserde_tokenstream-a3f756c2f035b223.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern usdt_impl=/work/oxidecomputer/crucible/target/debug/deps/libusdt_impl-aa84cb563c29877d.rlib --extern proc_macro --cap-lints allow`
   Compiling slog-bunyan v2.4.0
     Running `rustc --crate-name slog_bunyan --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-bunyan-2.4.0/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=dce051a6775f1d99 -C extra-filename=-dce051a6775f1d99 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern hostname=/work/oxidecomputer/crucible/target/debug/deps/libhostname-4a0f8b1a56e5681a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_json=/work/oxidecomputer/crucible/target/debug/deps/libslog_json-d8408f8f3a6dd5b7.rmeta --extern time=/work/oxidecomputer/crucible/target/debug/deps/libtime-9b604407a0d52f86.rmeta --cap-lints allow`
   Compiling toml v0.7.6
     Running `rustc --crate-name toml --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/toml-0.7.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="display"' --cfg 'feature="parse"' -C metadata=de0eb3fcc3b95b5c -C extra-filename=-de0eb3fcc3b95b5c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_spanned=/work/oxidecomputer/crucible/target/debug/deps/libserde_spanned-92d13640a4d5d836.rmeta --extern toml_datetime=/work/oxidecomputer/crucible/target/debug/deps/libtoml_datetime-c003080b79dfe49d.rmeta --extern toml_edit=/work/oxidecomputer/crucible/target/debug/deps/libtoml_edit-29ed73c573391add.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-69da610a4d2f950f/build-script-build`
     Running `rustc --crate-name libz_sys --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libz-sys-1.1.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="libc"' -C metadata=a3111f279c2174e3 -C extra-filename=-a3111f279c2174e3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow -L native=/usr/lib/amd64 -l z`
   Compiling errno v0.2.8
     Running `rustc --crate-name errno /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/errno-0.2.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=af3769d2acbbbf20 -C extra-filename=-af3769d2acbbbf20 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling usdt v0.3.5
     Running `rustc --crate-name usdt --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/usdt-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="asm"' --cfg 'feature="default"' --cfg 'feature="dtrace-parser"' -C metadata=86bb76e3b8fcea87 -C extra-filename=-86bb76e3b8fcea87 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dtrace_parser=/work/oxidecomputer/crucible/target/debug/deps/libdtrace_parser-647a421ab06e4ff3.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern usdt_attr_macro=/work/oxidecomputer/crucible/target/debug/deps/libusdt_attr_macro-1e49091236cd57b8.so --extern usdt_impl=/work/oxidecomputer/crucible/target/debug/deps/libusdt_impl-aa84cb563c29877d.rmeta --extern usdt_macro=/work/oxidecomputer/crucible/target/debug/deps/libusdt_macro-0ea5e72aa4bb61c5.so --cap-lints allow`
   Compiling utf8parse v0.2.1
   Compiling vergen v8.2.4
     Running `rustc --crate-name utf8parse --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/utf8parse-0.2.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=7be01c1e08c14495 -C extra-filename=-7be01c1e08c14495 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name build_script_build --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/vergen-8.2.4/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="cargo"' --cfg 'feature="default"' --cfg 'feature="git"' --cfg 'feature="git2"' --cfg 'feature="git2-rs"' --cfg 'feature="rustc"' --cfg 'feature="rustc_version"' --cfg 'feature="time"' -C metadata=43caef821611c916 -C extra-filename=-43caef821611c916 --out-dir /work/oxidecomputer/crucible/target/debug/build/vergen-43caef821611c916 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rustversion=/work/oxidecomputer/crucible/target/debug/deps/librustversion-ae2ca065d7ce77a2.so --cap-lints allow`
   Compiling memoffset v0.7.1
   Compiling num-integer v0.1.44
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/memoffset-0.7.1/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' -C metadata=4019f7a15f0e6805 -C extra-filename=-4019f7a15f0e6805 --out-dir /work/oxidecomputer/crucible/target/debug/build/memoffset-4019f7a15f0e6805 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-integer-0.1.44/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="std"' -C metadata=b9de1f088f036bd8 -C extra-filename=-b9de1f088f036bd8 --out-dir /work/oxidecomputer/crucible/target/debug/build/num-integer-b9de1f088f036bd8 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
   Compiling vcpkg v0.2.15
     Running `rustc --crate-name vcpkg /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/vcpkg-0.2.15/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=768a328be6bdd65b -C extra-filename=-768a328be6bdd65b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling rustc_version v0.4.0
     Running `rustc --crate-name rustc_version --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustc_version-0.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=201ef6100eba532b -C extra-filename=-201ef6100eba532b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern semver=/work/oxidecomputer/crucible/target/debug/deps/libsemver-8c1c5827befd93e7.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/vergen-43caef821611c916/build-script-build`
   Compiling rand_core v0.3.1
     Running `rustc --crate-name rand_core /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_core-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=588d7d54223a4bae -C extra-filename=-588d7d54223a4bae --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-cf5252fd1c1f594a.rmeta --cap-lints allow`
   Compiling slog-dtrace v0.2.3
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-dtrace-0.2.3/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=a361b6965b7c4d75 -C extra-filename=-a361b6965b7c4d75 --out-dir /work/oxidecomputer/crucible/target/debug/build/slog-dtrace-a361b6965b7c4d75 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern version_check=/work/oxidecomputer/crucible/target/debug/deps/libversion_check-23142ed5bf6178a0.rlib --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/memoffset-4019f7a15f0e6805/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/num-integer-b9de1f088f036bd8/build-script-build`
   Compiling static_assertions v1.1.0
     Running `rustc --crate-name static_assertions /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/static_assertions-1.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=457e7de98121159f -C extra-filename=-457e7de98121159f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling rustix v0.36.5
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustix-0.36.5/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="io-lifetimes"' --cfg 'feature="libc"' --cfg 'feature="std"' --cfg 'feature="termios"' --cfg 'feature="use-libc-auxv"' -C metadata=40dacc7b426369ca -C extra-filename=-40dacc7b426369ca --out-dir /work/oxidecomputer/crucible/target/debug/build/rustix-40dacc7b426369ca -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling ident_case v1.0.1
     Running `rustc --crate-name ident_case /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ident_case-1.0.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=90a2253e7db4d178 -C extra-filename=-90a2253e7db4d178 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name num_integer /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-integer-0.1.44/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=ada2de842cf2fef0 -C extra-filename=-ada2de842cf2fef0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow --cfg has_i128`
     Running `/work/oxidecomputer/crucible/target/debug/build/slog-dtrace-a361b6965b7c4d75/build-script-build`
     Running `rustc --crate-name memoffset /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/memoffset-0.7.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=7efa3f7abe9851cf -C extra-filename=-7efa3f7abe9851cf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg tuple_ty --cfg allow_clippy --cfg maybe_uninit --cfg doctests --cfg raw_ref_macros`
   Compiling darling_core v0.20.1
     Running `rustc --crate-name darling_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/darling_core-0.20.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="strsim"' --cfg 'feature="suggestions"' -C metadata=9504cdbd254aa6a2 -C extra-filename=-9504cdbd254aa6a2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern ident_case=/work/oxidecomputer/crucible/target/debug/deps/libident_case-90a2253e7db4d178.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern strsim=/work/oxidecomputer/crucible/target/debug/deps/libstrsim-3314ed6444b5bb69.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rmeta --cap-lints allow`
   Compiling tokio-util v0.7.3
     Running `rustc --crate-name tokio_util --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-util-0.7.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="codec"' --cfg 'feature="default"' --cfg 'feature="io"' --cfg 'feature="tracing"' -C metadata=279b3765a2b5aad1 -C extra-filename=-279b3765a2b5aad1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_sink=/work/oxidecomputer/crucible/target/debug/deps/libfutures_sink-a6d6ed8a846c5f8a.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --cap-lints allow`
   Compiling tokio-native-tls v0.3.0
     Running `rustc --crate-name tokio_native_tls --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-native-tls-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f56aba82a642e205 -C extra-filename=-f56aba82a642e205 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern native_tls=/work/oxidecomputer/crucible/target/debug/deps/libnative_tls-320c05ab5bbd33c9.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow -L native=/usr/ssl-3/lib/amd64`
     Running `/work/oxidecomputer/crucible/target/debug/build/rustix-40dacc7b426369ca/build-script-build`
   Compiling num-bigint v0.2.6
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-bigint-0.2.6/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="std"' -C metadata=7d556b8a17e8e692 -C extra-filename=-7d556b8a17e8e692 --out-dir /work/oxidecomputer/crucible/target/debug/build/num-bigint-7d556b8a17e8e692 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
   Compiling camino v1.1.4
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/camino-1.1.4/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="serde"' --cfg 'feature="serde1"' -C metadata=e2fc3fcdbebb7f61 -C extra-filename=-e2fc3fcdbebb7f61 --out-dir /work/oxidecomputer/crucible/target/debug/build/camino-e2fc3fcdbebb7f61 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling libsqlite3-sys v0.26.0
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libsqlite3-sys-0.26.0/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="min_sqlite_version_3_14_0"' --cfg 'feature="pkg-config"' --cfg 'feature="vcpkg"' -C metadata=9a39b2f8aed45f72 -C extra-filename=-9a39b2f8aed45f72 --out-dir /work/oxidecomputer/crucible/target/debug/build/libsqlite3-sys-9a39b2f8aed45f72 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern pkg_config=/work/oxidecomputer/crucible/target/debug/deps/libpkg_config-e05c47386f4bdcc0.rlib --extern vcpkg=/work/oxidecomputer/crucible/target/debug/deps/libvcpkg-768a328be6bdd65b.rlib --cap-lints allow`
   Compiling h2 v0.3.20
     Running `rustc --crate-name h2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/h2-0.3.20/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=3e2d8390f23dd48a -C extra-filename=-3e2d8390f23dd48a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_sink=/work/oxidecomputer/crucible/target/debug/deps/libfutures_sink-a6d6ed8a846c5f8a.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern slab=/work/oxidecomputer/crucible/target/debug/deps/libslab-5b7c79e345d6363e.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --cap-lints allow`
   Compiling semver v0.1.20
     Running `rustc --crate-name semver /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/semver-0.1.20/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=395e5508bfb3cbcf -C extra-filename=-395e5508bfb3cbcf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/libsqlite3-sys-9a39b2f8aed45f72/build-script-build`
   Compiling bitflags v2.3.3
     Running `rustc --crate-name bitflags --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bitflags-2.3.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=109244799287a8c3 -C extra-filename=-109244799287a8c3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name libsqlite3_sys --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libsqlite3-sys-0.26.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="min_sqlite_version_3_14_0"' --cfg 'feature="pkg-config"' --cfg 'feature="vcpkg"' -C metadata=d6c034ac2a660d5f -C extra-filename=-d6c034ac2a660d5f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow -L native=/usr/lib/amd64 -l sqlite3`
   Compiling rustc_version v0.1.7
     Running `rustc --crate-name rustc_version /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustc_version-0.1.7/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=478fd37c27f4f33b -C extra-filename=-478fd37c27f4f33b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern semver=/work/oxidecomputer/crucible/target/debug/deps/libsemver-395e5508bfb3cbcf.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/camino-e2fc3fcdbebb7f61/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/num-bigint-7d556b8a17e8e692/build-script-build`
     Running `rustc --crate-name rustix --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustix-0.36.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="io-lifetimes"' --cfg 'feature="libc"' --cfg 'feature="std"' --cfg 'feature="termios"' --cfg 'feature="use-libc-auxv"' -C metadata=06529533a7816508 -C extra-filename=-06529533a7816508 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern libc_errno=/work/oxidecomputer/crucible/target/debug/deps/liberrno-af3769d2acbbbf20.rmeta --extern io_lifetimes=/work/oxidecomputer/crucible/target/debug/deps/libio_lifetimes-df7ee936a2a5cbac.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow --cfg libc`
   Compiling nix v0.26.2
     Running `rustc --crate-name nix --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/nix-0.26.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="acct"' --cfg 'feature="aio"' --cfg 'feature="default"' --cfg 'feature="dir"' --cfg 'feature="env"' --cfg 'feature="event"' --cfg 'feature="feature"' --cfg 'feature="fs"' --cfg 'feature="hostname"' --cfg 'feature="inotify"' --cfg 'feature="ioctl"' --cfg 'feature="kmod"' --cfg 'feature="memoffset"' --cfg 'feature="mman"' --cfg 'feature="mount"' --cfg 'feature="mqueue"' --cfg 'feature="net"' --cfg 'feature="personality"' --cfg 'feature="pin-utils"' --cfg 'feature="poll"' --cfg 'feature="process"' --cfg 'feature="pthread"' --cfg 'feature="ptrace"' --cfg 'feature="quota"' --cfg 'feature="reboot"' --cfg 'feature="resource"' --cfg 'feature="sched"' --cfg 'feature="signal"' --cfg 'feature="socket"' --cfg 'feature="term"' --cfg 'feature="time"' --cfg 'feature="ucontext"' --cfg 'feature="uio"' --cfg 'feature="user"' --cfg 'feature="zerocopy"' -C metadata=3635aff9412bf811 -C extra-filename=-3635aff9412bf811 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern memoffset=/work/oxidecomputer/crucible/target/debug/deps/libmemoffset-7efa3f7abe9851cf.rmeta --extern pin_utils=/work/oxidecomputer/crucible/target/debug/deps/libpin_utils-bcfb754cd1ab8c67.rmeta --extern static_assertions=/work/oxidecomputer/crucible/target/debug/deps/libstatic_assertions-457e7de98121159f.rmeta --cap-lints allow`
     Running `rustc --crate-name slog_dtrace --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/slog-dtrace-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=8024beacfb95325b -C extra-filename=-8024beacfb95325b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rmeta --cap-lints allow`
   Compiling anstyle-parse v0.2.0
     Running `rustc --crate-name anstyle_parse --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anstyle-parse-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="utf8"' -C metadata=1cddf38a13db6213 -C extra-filename=-1cddf38a13db6213 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern utf8parse=/work/oxidecomputer/crucible/target/debug/deps/libutf8parse-7be01c1e08c14495.rmeta --cap-lints allow`
   Compiling rand_pcg v0.1.2
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_pcg-0.1.2/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=0d95134e750a04ec -C extra-filename=-0d95134e750a04ec --out-dir /work/oxidecomputer/crucible/target/debug/build/rand_pcg-0d95134e750a04ec -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-0a34a0aae5d9487a.rlib --cap-lints allow`
   Compiling rand_chacha v0.1.1
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_chacha-0.1.1/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=f5c658d3512bc715 -C extra-filename=-f5c658d3512bc715 --out-dir /work/oxidecomputer/crucible/target/debug/build/rand_chacha-f5c658d3512bc715 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-0a34a0aae5d9487a.rlib --cap-lints allow`
   Compiling async-stream-impl v0.3.5
     Running `rustc --crate-name async_stream_impl --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stream-impl-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=499322f076b313bc -C extra-filename=-499322f076b313bc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling hashlink v0.8.0
     Running `rustc --crate-name hashlink --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hashlink-0.8.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=20d837d226474f7f -C extra-filename=-20d837d226474f7f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern hashbrown=/work/oxidecomputer/crucible/target/debug/deps/libhashbrown-3ad6614047c487f9.rmeta --cap-lints allow`
   Compiling num-rational v0.2.4
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-rational-0.2.4/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="bigint"' --cfg 'feature="num-bigint"' --cfg 'feature="std"' -C metadata=ff937c196b938c41 -C extra-filename=-ff937c196b938c41 --out-dir /work/oxidecomputer/crucible/target/debug/build/num-rational-ff937c196b938c41 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
   Compiling num-complex v0.2.4
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-complex-0.2.4/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="std"' -C metadata=4def68cf859a947c -C extra-filename=-4def68cf859a947c --out-dir /work/oxidecomputer/crucible/target/debug/build/num-complex-4def68cf859a947c -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
   Compiling num-iter v0.1.42
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-iter-0.1.42/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="std"' -C metadata=4bc024ee16b2fd63 -C extra-filename=-4bc024ee16b2fd63 --out-dir /work/oxidecomputer/crucible/target/debug/build/num-iter-4bc024ee16b2fd63 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
   Compiling anstyle-query v1.0.0
     Running `rustc --crate-name anstyle_query --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anstyle-query-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f6b83c0a54455ea0 -C extra-filename=-f6b83c0a54455ea0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling fallible-streaming-iterator v0.1.9
     Running `rustc --crate-name fallible_streaming_iterator /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fallible-streaming-iterator-0.1.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=95608b7ccd5e88f6 -C extra-filename=-95608b7ccd5e88f6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling colorchoice v1.0.0
     Running `rustc --crate-name colorchoice --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/colorchoice-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=88cb686aa85d1c3b -C extra-filename=-88cb686aa85d1c3b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling paste v1.0.14
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/paste-1.0.14/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=9a388a178f36d25e -C extra-filename=-9a388a178f36d25e --out-dir /work/oxidecomputer/crucible/target/debug/build/paste-9a388a178f36d25e -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling anstyle v1.0.0
     Running `rustc --crate-name anstyle --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anstyle-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=b22d2b8ea1e30552 -C extra-filename=-b22d2b8ea1e30552 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling rusqlite v0.29.0
     Running `rustc --crate-name rusqlite --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rusqlite-0.29.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e4d2316a88b06837 -C extra-filename=-e4d2316a88b06837 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-109244799287a8c3.rmeta --extern fallible_iterator=/work/oxidecomputer/crucible/target/debug/deps/libfallible_iterator-2f475e2aff163c98.rmeta --extern fallible_streaming_iterator=/work/oxidecomputer/crucible/target/debug/deps/libfallible_streaming_iterator-95608b7ccd5e88f6.rmeta --extern hashlink=/work/oxidecomputer/crucible/target/debug/deps/libhashlink-20d837d226474f7f.rmeta --extern libsqlite3_sys=/work/oxidecomputer/crucible/target/debug/deps/liblibsqlite3_sys-d6c034ac2a660d5f.rmeta --extern smallvec=/work/oxidecomputer/crucible/target/debug/deps/libsmallvec-397f26bd8c84e528.rmeta --cap-lints allow -L native=/usr/lib/amd64`
   Compiling darling_macro v0.20.1
     Running `rustc --crate-name darling_macro --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/darling_macro-0.20.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=9054a39daee42f78 -C extra-filename=-9054a39daee42f78 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern darling_core=/work/oxidecomputer/crucible/target/debug/deps/libdarling_core-9504cdbd254aa6a2.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling hyper v0.14.27
     Running `rustc --crate-name hyper --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hyper-0.14.27/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="client"' --cfg 'feature="default"' --cfg 'feature="full"' --cfg 'feature="h2"' --cfg 'feature="http1"' --cfg 'feature="http2"' --cfg 'feature="runtime"' --cfg 'feature="server"' --cfg 'feature="socket2"' --cfg 'feature="stream"' --cfg 'feature="tcp"' -C metadata=dad943d3b7cc33e9 -C extra-filename=-dad943d3b7cc33e9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern h2=/work/oxidecomputer/crucible/target/debug/deps/libh2-3e2d8390f23dd48a.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern http_body=/work/oxidecomputer/crucible/target/debug/deps/libhttp_body-bb1d69dd918c127f.rmeta --extern httparse=/work/oxidecomputer/crucible/target/debug/deps/libhttparse-59406412a39ce707.rmeta --extern httpdate=/work/oxidecomputer/crucible/target/debug/deps/libhttpdate-e3ef82a990113a54.rmeta --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern socket2=/work/oxidecomputer/crucible/target/debug/deps/libsocket2-3c3e3607c1c6d64e.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tower_service=/work/oxidecomputer/crucible/target/debug/deps/libtower_service-51da71f2ad5117ee.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --extern want=/work/oxidecomputer/crucible/target/debug/deps/libwant-5737a0d118420ef7.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/paste-9a388a178f36d25e/build-script-build`
   Compiling anstream v0.5.0
     Running `rustc --crate-name anstream --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anstream-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="auto"' --cfg 'feature="default"' --cfg 'feature="wincon"' -C metadata=ca70a88d33d17944 -C extra-filename=-ca70a88d33d17944 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anstyle=/work/oxidecomputer/crucible/target/debug/deps/libanstyle-b22d2b8ea1e30552.rmeta --extern anstyle_parse=/work/oxidecomputer/crucible/target/debug/deps/libanstyle_parse-1cddf38a13db6213.rmeta --extern anstyle_query=/work/oxidecomputer/crucible/target/debug/deps/libanstyle_query-f6b83c0a54455ea0.rmeta --extern colorchoice=/work/oxidecomputer/crucible/target/debug/deps/libcolorchoice-88cb686aa85d1c3b.rmeta --extern utf8parse=/work/oxidecomputer/crucible/target/debug/deps/libutf8parse-7be01c1e08c14495.rmeta --cap-lints allow`
   Compiling async-stream v0.3.5
     Running `rustc --crate-name async_stream --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stream-0.3.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0486f21173e73f9c -C extra-filename=-0486f21173e73f9c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_stream_impl=/work/oxidecomputer/crucible/target/debug/deps/libasync_stream_impl-499322f076b313bc.so --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --cap-lints allow`
   Compiling darling v0.20.1
     Running `rustc --crate-name darling --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/darling-0.20.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="suggestions"' -C metadata=00ecdd47ee30dd62 -C extra-filename=-00ecdd47ee30dd62 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern darling_core=/work/oxidecomputer/crucible/target/debug/deps/libdarling_core-9504cdbd254aa6a2.rmeta --extern darling_macro=/work/oxidecomputer/crucible/target/debug/deps/libdarling_macro-9054a39daee42f78.so --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/num-iter-4bc024ee16b2fd63/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/num-complex-4def68cf859a947c/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/num-rational-ff937c196b938c41/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/rand_chacha-f5c658d3512bc715/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/rand_pcg-0d95134e750a04ec/build-script-build`
   Compiling terminal_size v0.2.5
     Running `rustc --crate-name terminal_size --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/terminal_size-0.2.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0b293d2f7e4e73b7 -C extra-filename=-0b293d2f7e4e73b7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rustix=/work/oxidecomputer/crucible/target/debug/deps/librustix-06529533a7816508.rmeta --cap-lints allow`
     Running `rustc --crate-name num_bigint /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-bigint-0.2.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=fb487451ba2d3918 -C extra-filename=-fb487451ba2d3918 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_integer=/work/oxidecomputer/crucible/target/debug/deps/libnum_integer-ada2de842cf2fef0.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow --cfg has_i128`
   Compiling newtype_derive v0.1.6
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/newtype_derive-0.1.6/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=b4faed40373b7a32 -C extra-filename=-b4faed40373b7a32 --out-dir /work/oxidecomputer/crucible/target/debug/build/newtype_derive-b4faed40373b7a32 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rustc_version=/work/oxidecomputer/crucible/target/debug/deps/librustc_version-478fd37c27f4f33b.rlib --cap-lints allow`
     Running `rustc --crate-name camino --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/camino-1.1.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="serde"' --cfg 'feature="serde1"' -C metadata=45f0f4a2c258f934 -C extra-filename=-45f0f4a2c258f934 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow --cfg path_buf_capacity --cfg shrink_to --cfg try_reserve_2 --cfg path_buf_deref_mut`
   Compiling twox-hash v1.6.3
     Running `rustc --crate-name twox_hash --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/twox-hash-1.6.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="rand"' --cfg 'feature="std"' -C metadata=9f5dd4f7319ca539 -C extra-filename=-9f5dd4f7319ca539 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern static_assertions=/work/oxidecomputer/crucible/target/debug/deps/libstatic_assertions-457e7de98121159f.rmeta --cap-lints allow`
   Compiling hex v0.4.3
     Running `rustc --crate-name hex --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hex-0.4.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="std"' -C metadata=6531b11cb72de3e5 -C extra-filename=-6531b11cb72de3e5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling sha2 v0.10.2
     Running `rustc --crate-name sha2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/sha2-0.10.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=9b09b4b286e2cb62 -C extra-filename=-9b09b4b286e2cb62 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern cpufeatures=/work/oxidecomputer/crucible/target/debug/deps/libcpufeatures-6e25ef20a8fa218f.rmeta --extern digest=/work/oxidecomputer/crucible/target/debug/deps/libdigest-d0d34473efe105d1.rmeta --cap-lints allow`
   Compiling rand v0.6.5
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand-0.6.5/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="rand_os"' --cfg 'feature="std"' -C metadata=f0b68220a1e4759e -C extra-filename=-f0b68220a1e4759e --out-dir /work/oxidecomputer/crucible/target/debug/build/rand-f0b68220a1e4759e -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-0a34a0aae5d9487a.rlib --cap-lints allow`
   Compiling md-5 v0.10.1
     Running `rustc --crate-name md5 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/md-5-0.10.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=161e8167d58df2d6 -C extra-filename=-161e8167d58df2d6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern digest=/work/oxidecomputer/crucible/target/debug/deps/libdigest-d0d34473efe105d1.rmeta --cap-lints allow`
   Compiling hmac v0.12.1
     Running `rustc --crate-name hmac --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hmac-0.12.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=dbc6bea790b908e1 -C extra-filename=-dbc6bea790b908e1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern digest=/work/oxidecomputer/crucible/target/debug/deps/libdigest-d0d34473efe105d1.rmeta --cap-lints allow`
   Compiling structmeta-derive v0.1.5
     Running `rustc --crate-name structmeta_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/structmeta-derive-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=fab85507a465b1dd -C extra-filename=-fab85507a465b1dd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern proc_macro --cap-lints allow`
   Compiling stringprep v0.1.2
     Running `rustc --crate-name stringprep /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/stringprep-0.1.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ca9dbc365b4e987e -C extra-filename=-ca9dbc365b4e987e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern unicode_bidi=/work/oxidecomputer/crucible/target/debug/deps/libunicode_bidi-2c825f5beb05a037.rmeta --extern unicode_normalization=/work/oxidecomputer/crucible/target/debug/deps/libunicode_normalization-3df8261a03d4248e.rmeta --cap-lints allow`
   Compiling dropshot v0.9.1-dev (https://github.com/oxidecomputer/dropshot?branch=main#aca6de3c)
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/aca6de3/dropshot/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="usdt"' --cfg 'feature="usdt-probes"' -C metadata=713c18277b5bcf4b -C extra-filename=-713c18277b5bcf4b --out-dir /work/oxidecomputer/crucible/target/debug/build/dropshot-713c18277b5bcf4b -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern version_check=/work/oxidecomputer/crucible/target/debug/deps/libversion_check-23142ed5bf6178a0.rlib --cap-lints allow`
   Compiling regex-syntax v0.6.28
     Running `rustc --crate-name regex_syntax --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/regex-syntax-0.6.28/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="unicode"' --cfg 'feature="unicode-age"' --cfg 'feature="unicode-bool"' --cfg 'feature="unicode-case"' --cfg 'feature="unicode-gencat"' --cfg 'feature="unicode-perl"' --cfg 'feature="unicode-script"' --cfg 'feature="unicode-segment"' -C metadata=b9593ef3338880de -C extra-filename=-b9593ef3338880de --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling clap_lex v0.5.0
     Running `rustc --crate-name clap_lex --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/clap_lex-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=cd6fc5d5239978f4 -C extra-filename=-cd6fc5d5239978f4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling atomic-waker v1.1.1
     Running `rustc --crate-name atomic_waker --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/atomic-waker-1.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=2b745687caafccb6 -C extra-filename=-2b745687caafccb6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling base64 v0.13.1
     Running `rustc --crate-name base64 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/base64-0.13.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=480956047b7063a4 -C extra-filename=-480956047b7063a4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling waitgroup v0.1.2
     Running `rustc --crate-name waitgroup --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/waitgroup-0.1.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=db859ead02bd709e -C extra-filename=-db859ead02bd709e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern atomic_waker=/work/oxidecomputer/crucible/target/debug/deps/libatomic_waker-2b745687caafccb6.rmeta --cap-lints allow`
   Compiling hyper-tls v0.5.0
     Running `rustc --crate-name hyper_tls --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hyper-tls-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=2dbf57f91f681e2c -C extra-filename=-2dbf57f91f681e2c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern native_tls=/work/oxidecomputer/crucible/target/debug/deps/libnative_tls-320c05ab5bbd33c9.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_native_tls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_native_tls-f56aba82a642e205.rmeta --cap-lints allow -L native=/usr/ssl-3/lib/amd64`
   Compiling postgres-protocol v0.6.4
     Running `rustc --crate-name postgres_protocol --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/postgres-protocol-0.6.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=2544e50ad0920d44 -C extra-filename=-2544e50ad0920d44 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-480956047b7063a4.rmeta --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern fallible_iterator=/work/oxidecomputer/crucible/target/debug/deps/libfallible_iterator-2f475e2aff163c98.rmeta --extern hmac=/work/oxidecomputer/crucible/target/debug/deps/libhmac-dbc6bea790b908e1.rmeta --extern md5=/work/oxidecomputer/crucible/target/debug/deps/libmd5-161e8167d58df2d6.rmeta --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rmeta --extern stringprep=/work/oxidecomputer/crucible/target/debug/deps/libstringprep-ca9dbc365b4e987e.rmeta --cap-lints allow`
   Compiling clap_builder v4.4.0
     Running `rustc --crate-name clap_builder --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/clap_builder-4.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="color"' --cfg 'feature="env"' --cfg 'feature="error-context"' --cfg 'feature="help"' --cfg 'feature="std"' --cfg 'feature="suggestions"' --cfg 'feature="usage"' --cfg 'feature="wrap_help"' -C metadata=f6e4b2a8b78f1399 -C extra-filename=-f6e4b2a8b78f1399 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anstream=/work/oxidecomputer/crucible/target/debug/deps/libanstream-ca70a88d33d17944.rmeta --extern anstyle=/work/oxidecomputer/crucible/target/debug/deps/libanstyle-b22d2b8ea1e30552.rmeta --extern clap_lex=/work/oxidecomputer/crucible/target/debug/deps/libclap_lex-cd6fc5d5239978f4.rmeta --extern strsim=/work/oxidecomputer/crucible/target/debug/deps/libstrsim-3314ed6444b5bb69.rmeta --extern terminal_size=/work/oxidecomputer/crucible/target/debug/deps/libterminal_size-0b293d2f7e4e73b7.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/dropshot-713c18277b5bcf4b/build-script-build`
     Running `rustc --crate-name num_rational /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-rational-0.2.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bigint"' --cfg 'feature="num-bigint"' --cfg 'feature="std"' -C metadata=ff1997054aaa4c62 -C extra-filename=-ff1997054aaa4c62 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_bigint=/work/oxidecomputer/crucible/target/debug/deps/libnum_bigint-fb487451ba2d3918.rmeta --extern num_integer=/work/oxidecomputer/crucible/target/debug/deps/libnum_integer-ada2de842cf2fef0.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow --cfg has_i128 --cfg has_const_fn`
     Running `/work/oxidecomputer/crucible/target/debug/build/rand-f0b68220a1e4759e/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/newtype_derive-b4faed40373b7a32/build-script-build`
     Running `rustc --crate-name rand_pcg /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_pcg-0.1.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=64095680c19f1d75 -C extra-filename=-64095680c19f1d75 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-cf5252fd1c1f594a.rmeta --cap-lints allow --cfg rustc_1_26`
     Running `rustc --crate-name rand_chacha /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_chacha-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=60a86e6d4f82a03e -C extra-filename=-60a86e6d4f82a03e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-588d7d54223a4bae.rmeta --cap-lints allow --cfg rustc_1_26`
     Running `rustc --crate-name num_complex /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-complex-0.2.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=b0ad3f1350de6722 -C extra-filename=-b0ad3f1350de6722 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow --cfg has_i128 --cfg has_const_fn`
     Running `rustc --crate-name num_iter /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-iter-0.1.42/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=ea5f21eb64e222e3 -C extra-filename=-ea5f21eb64e222e3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_integer=/work/oxidecomputer/crucible/target/debug/deps/libnum_integer-ada2de842cf2fef0.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow --cfg has_i128`
   Compiling structmeta v0.1.5
     Running `rustc --crate-name structmeta --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/structmeta-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=478a2e76026e2d21 -C extra-filename=-478a2e76026e2d21 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern structmeta_derive=/work/oxidecomputer/crucible/target/debug/deps/libstructmeta_derive-fab85507a465b1dd.so --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rmeta --cap-lints allow`
   Compiling serde_with_macros v2.3.3
     Running `rustc --crate-name serde_with_macros --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_with_macros-2.3.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=0356ef5a55ed4c76 -C extra-filename=-0356ef5a55ed4c76 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern darling=/work/oxidecomputer/crucible/target/debug/deps/libdarling-00ecdd47ee30dd62.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
     Running `rustc --crate-name paste --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/paste-1.0.14/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=251489637fc3d2bc -C extra-filename=-251489637fc3d2bc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro --cap-lints allow`
   Compiling rand_xorshift v0.1.1
     Running `rustc --crate-name rand_xorshift /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_xorshift-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=516db51379fddd21 -C extra-filename=-516db51379fddd21 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-588d7d54223a4bae.rmeta --cap-lints allow`
   Compiling rand_hc v0.1.0
     Running `rustc --crate-name rand_hc /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_hc-0.1.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e624ce70fff78cce -C extra-filename=-e624ce70fff78cce --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-588d7d54223a4bae.rmeta --cap-lints allow`
   Compiling rand_isaac v0.1.1
     Running `rustc --crate-name rand_isaac /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_isaac-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=2a69a705c11dd4e8 -C extra-filename=-2a69a705c11dd4e8 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-588d7d54223a4bae.rmeta --cap-lints allow`
   Compiling dropshot_endpoint v0.9.1-dev (https://github.com/oxidecomputer/dropshot?branch=main#aca6de3c)
     Running `rustc --crate-name dropshot_endpoint --edition=2018 /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/aca6de3/dropshot_endpoint/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=1ff3a3dd0352c250 -C extra-filename=-1ff3a3dd0352c250 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_tokenstream=/work/oxidecomputer/crucible/target/debug/deps/libserde_tokenstream-2a83027836bfd0fb.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling rand_jitter v0.1.4
     Running `rustc --crate-name rand_jitter /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_jitter-0.1.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=4375ed928e3b252c -C extra-filename=-4375ed928e3b252c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-cf5252fd1c1f594a.rmeta --cap-lints allow`
   Compiling rand_os v0.1.3
     Running `rustc --crate-name rand_os /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_os-0.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5662c3e15991676f -C extra-filename=-5662c3e15991676f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-cf5252fd1c1f594a.rmeta --cap-lints allow`
   Compiling serde_path_to_error v0.1.14
     Running `rustc --crate-name serde_path_to_error --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_path_to_error-0.1.14/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f8fffad8b554a310 -C extra-filename=-f8fffad8b554a310 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling sha1 v0.10.5
     Running `rustc --crate-name sha1 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/sha1-0.10.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=8fdbd7a715f3bef1 -C extra-filename=-8fdbd7a715f3bef1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern cpufeatures=/work/oxidecomputer/crucible/target/debug/deps/libcpufeatures-6e25ef20a8fa218f.rmeta --extern digest=/work/oxidecomputer/crucible/target/debug/deps/libdigest-d0d34473efe105d1.rmeta --cap-lints allow`
   Compiling phf_shared v0.10.0
     Running `rustc --crate-name phf_shared --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf_shared-0.10.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=ded46b90978b786e -C extra-filename=-ded46b90978b786e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern siphasher=/work/oxidecomputer/crucible/target/debug/deps/libsiphasher-2eb27360f66646f3.rmeta --cap-lints allow`
   Compiling strum_macros v0.25.2
     Running `rustc --crate-name strum_macros --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/strum_macros-0.25.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=bc907f623478289d -C extra-filename=-bc907f623478289d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern heck=/work/oxidecomputer/crucible/target/debug/deps/libheck-29af75c938b110f7.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern rustversion=/work/oxidecomputer/crucible/target/debug/deps/librustversion-ae2ca065d7ce77a2.so --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling clap_derive v4.4.0
     Running `rustc --crate-name clap_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/clap_derive-4.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="default"' -C metadata=09787059b4668753 -C extra-filename=-09787059b4668753 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern heck=/work/oxidecomputer/crucible/target/debug/deps/libheck-29af75c938b110f7.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling debug-ignore v1.0.5
     Running `rustc --crate-name debug_ignore /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/debug-ignore-1.0.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=2303f500fcbc7093 -C extra-filename=-2303f500fcbc7093 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling either v1.6.1
     Running `rustc --crate-name either /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/either-1.6.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="use_std"' -C metadata=e075e5427abad613 -C extra-filename=-e075e5427abad613 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling fixedbitset v0.4.1
     Running `rustc --crate-name fixedbitset /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fixedbitset-0.4.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c5fa142e7e51f766 -C extra-filename=-c5fa142e7e51f766 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling serde_with v2.3.3
     Running `rustc --crate-name serde_with --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/serde_with-2.3.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="macros"' --cfg 'feature="std"' -C metadata=4f9ddd30b380d6cf -C extra-filename=-4f9ddd30b380d6cf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_with_macros=/work/oxidecomputer/crucible/target/debug/deps/libserde_with_macros-0356ef5a55ed4c76.so --cap-lints allow`
   Compiling petgraph v0.6.3
     Running `rustc --crate-name petgraph --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/petgraph-0.6.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="graphmap"' --cfg 'feature="matrix_graph"' --cfg 'feature="serde"' --cfg 'feature="serde-1"' --cfg 'feature="serde_derive"' --cfg 'feature="stable_graph"' -C metadata=8e9556a1882bb1d5 -C extra-filename=-8e9556a1882bb1d5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern fixedbitset=/work/oxidecomputer/crucible/target/debug/deps/libfixedbitset-c5fa142e7e51f766.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_derive=/work/oxidecomputer/crucible/target/debug/deps/libserde_derive-3441a1f9756a6d5b.so --cap-lints allow`
   Compiling phf v0.10.1
     Running `rustc --crate-name phf --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf-0.10.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=692e1d997f5b6ce1 -C extra-filename=-692e1d997f5b6ce1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern phf_shared=/work/oxidecomputer/crucible/target/debug/deps/libphf_shared-ded46b90978b786e.rmeta --cap-lints allow`
     Running `rustc --crate-name rand /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand-0.6.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="rand_os"' --cfg 'feature="std"' -C metadata=d54af7a747f291ab -C extra-filename=-d54af7a747f291ab --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-60a86e6d4f82a03e.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-cf5252fd1c1f594a.rmeta --extern rand_hc=/work/oxidecomputer/crucible/target/debug/deps/librand_hc-e624ce70fff78cce.rmeta --extern rand_isaac=/work/oxidecomputer/crucible/target/debug/deps/librand_isaac-2a69a705c11dd4e8.rmeta --extern rand_jitter=/work/oxidecomputer/crucible/target/debug/deps/librand_jitter-4375ed928e3b252c.rmeta --extern rand_os=/work/oxidecomputer/crucible/target/debug/deps/librand_os-5662c3e15991676f.rmeta --extern rand_pcg=/work/oxidecomputer/crucible/target/debug/deps/librand_pcg-64095680c19f1d75.rmeta --extern rand_xorshift=/work/oxidecomputer/crucible/target/debug/deps/librand_xorshift-516db51379fddd21.rmeta --cap-lints allow --cfg rustc_1_25 --cfg rustc_1_26 --cfg rustc_1_27`
   Compiling parse-display-derive v0.7.0
     Running `rustc --crate-name parse_display_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/parse-display-derive-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=95e2e5d955c96bcf -C extra-filename=-95e2e5d955c96bcf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern regex=/work/oxidecomputer/crucible/target/debug/deps/libregex-f9e3a4eb3da387ce.rlib --extern regex_syntax=/work/oxidecomputer/crucible/target/debug/deps/libregex_syntax-b9593ef3338880de.rlib --extern structmeta=/work/oxidecomputer/crucible/target/debug/deps/libstructmeta-478a2e76026e2d21.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-837f9a049f77ca38.rlib --extern proc_macro --cap-lints allow`
   Compiling num v0.2.1
     Running `rustc --crate-name num /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-0.2.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="num-bigint"' --cfg 'feature="std"' -C metadata=1148753de5cf68fd -C extra-filename=-1148753de5cf68fd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_bigint=/work/oxidecomputer/crucible/target/debug/deps/libnum_bigint-fb487451ba2d3918.rmeta --extern num_complex=/work/oxidecomputer/crucible/target/debug/deps/libnum_complex-b0ad3f1350de6722.rmeta --extern num_integer=/work/oxidecomputer/crucible/target/debug/deps/libnum_integer-ada2de842cf2fef0.rmeta --extern num_iter=/work/oxidecomputer/crucible/target/debug/deps/libnum_iter-ea5f21eb64e222e3.rmeta --extern num_rational=/work/oxidecomputer/crucible/target/debug/deps/libnum_rational-ff1997054aaa4c62.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow`
     Running `rustc --crate-name newtype_derive /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/newtype_derive-0.1.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=fedf432d6b5251a4 -C extra-filename=-fedf432d6b5251a4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg op_assign`
   Compiling strum v0.25.0
     Running `rustc --crate-name strum --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/strum-0.25.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="std"' --cfg 'feature="strum_macros"' -C metadata=59ea3c6704348e58 -C extra-filename=-59ea3c6704348e58 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern strum_macros=/work/oxidecomputer/crucible/target/debug/deps/libstrum_macros-bc907f623478289d.so --cap-lints allow`
   Compiling postgres-types v0.2.3
     Running `rustc --crate-name postgres_types --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/postgres-types-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="chrono-04"' --cfg 'feature="uuid-1"' --cfg 'feature="with-chrono-0_4"' --cfg 'feature="with-uuid-1"' -C metadata=3d7a8ed523cd84fa -C extra-filename=-3d7a8ed523cd84fa --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern chrono_04=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern fallible_iterator=/work/oxidecomputer/crucible/target/debug/deps/libfallible_iterator-2f475e2aff163c98.rmeta --extern postgres_protocol=/work/oxidecomputer/crucible/target/debug/deps/libpostgres_protocol-2544e50ad0920d44.rmeta --extern uuid_1=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow`
   Compiling proc-macro-crate v1.3.1
     Running `rustc --crate-name proc_macro_crate --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/proc-macro-crate-1.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=3935aefc56866ee2 -C extra-filename=-3935aefc56866ee2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern toml_edit=/work/oxidecomputer/crucible/target/debug/deps/libtoml_edit-29ed73c573391add.rmeta --cap-lints allow`
   Compiling instant v0.1.12
     Running `rustc --crate-name instant --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/instant-0.1.12/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=3f52fffb2a7c0a97 -C extra-filename=-3f52fffb2a7c0a97 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --cap-lints allow`
   Compiling clap v4.4.0
     Running `rustc --crate-name clap --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/clap-4.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="color"' --cfg 'feature="default"' --cfg 'feature="derive"' --cfg 'feature="env"' --cfg 'feature="error-context"' --cfg 'feature="help"' --cfg 'feature="std"' --cfg 'feature="suggestions"' --cfg 'feature="usage"' --cfg 'feature="wrap_help"' -C metadata=49bc17aade028e79 -C extra-filename=-49bc17aade028e79 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern clap_builder=/work/oxidecomputer/crucible/target/debug/deps/libclap_builder-f6e4b2a8b78f1399.rmeta --extern clap_derive=/work/oxidecomputer/crucible/target/debug/deps/libclap_derive-09787059b4668753.so --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --cap-lints allow`
   Compiling backoff v0.4.0
     Running `rustc --crate-name backoff --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/backoff-0.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="futures"' --cfg 'feature="futures-core"' --cfg 'feature="pin-project-lite"' --cfg 'feature="tokio"' --cfg 'feature="tokio_1"' -C metadata=2bc4a2fd075cf434 -C extra-filename=-2bc4a2fd075cf434 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern getrandom=/work/oxidecomputer/crucible/target/debug/deps/libgetrandom-567199de146d617e.rmeta --extern instant=/work/oxidecomputer/crucible/target/debug/deps/libinstant-3f52fffb2a7c0a97.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern tokio_1=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow`
   Compiling tokio-postgres v0.7.6
     Running `rustc --crate-name tokio_postgres --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-postgres-0.7.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="runtime"' --cfg 'feature="with-chrono-0_4"' --cfg 'feature="with-uuid-1"' -C metadata=5628b93feb58339b -C extra-filename=-5628b93feb58339b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern fallible_iterator=/work/oxidecomputer/crucible/target/debug/deps/libfallible_iterator-2f475e2aff163c98.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern parking_lot=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot-970d5c0acece447c.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern phf=/work/oxidecomputer/crucible/target/debug/deps/libphf-692e1d997f5b6ce1.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern postgres_protocol=/work/oxidecomputer/crucible/target/debug/deps/libpostgres_protocol-2544e50ad0920d44.rmeta --extern postgres_types=/work/oxidecomputer/crucible/target/debug/deps/libpostgres_types-3d7a8ed523cd84fa.rmeta --extern socket2=/work/oxidecomputer/crucible/target/debug/deps/libsocket2-3c3e3607c1c6d64e.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --cap-lints allow`
   Compiling steno v0.4.0
     Running `rustc --crate-name steno --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/steno-0.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=d1d3ce30296926ad -C extra-filename=-d1d3ce30296926ad --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern newtype_derive=/work/oxidecomputer/crucible/target/debug/deps/libnewtype_derive-fedf432d6b5251a4.rmeta --extern petgraph=/work/oxidecomputer/crucible/target/debug/deps/libpetgraph-8e9556a1882bb1d5.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow`
   Compiling num_enum_derive v0.7.0
     Running `rustc --crate-name num_enum_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num_enum_derive-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no --cfg 'feature="proc-macro-crate"' --cfg 'feature="std"' -C metadata=2bef02e01d5b06d3 -C extra-filename=-2bef02e01d5b06d3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro_crate=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro_crate-3935aefc56866ee2.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling statistical v1.0.0
     Running `rustc --crate-name statistical /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/statistical-1.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f2809857d7fe1c91 -C extra-filename=-f2809857d7fe1c91 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num=/work/oxidecomputer/crucible/target/debug/deps/libnum-1148753de5cf68fd.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-d54af7a747f291ab.rmeta --cap-lints allow`
   Compiling serde_human_bytes v0.1.0 (http://github.com/oxidecomputer/serde_human_bytes?branch=main#0a097945)
     Running `rustc --crate-name serde_human_bytes --edition=2018 /home/build/.cargo/git/checkouts/serde_human_bytes-25cac76e2ef2d15c/0a09794/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=7f54d0fcbf9b36f3 -C extra-filename=-7f54d0fcbf9b36f3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling camino-tempfile v1.0.2
     Running `rustc --crate-name camino_tempfile --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/camino-tempfile-1.0.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=38b6a8d85c9dc0c0 -C extra-filename=-38b6a8d85c9dc0c0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern camino=/work/oxidecomputer/crucible/target/debug/deps/libcamino-45f0f4a2c258f934.rmeta --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rmeta --cap-lints allow`
   Compiling ipnetwork v0.20.0
     Running `rustc --crate-name ipnetwork --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ipnetwork-0.20.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="schemars"' --cfg 'feature="serde"' -C metadata=0e9e550a49db2c52 -C extra-filename=-0e9e550a49db2c52 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling macaddr v1.0.1
     Running `rustc --crate-name macaddr --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/macaddr-1.0.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="serde_std"' --cfg 'feature="std"' -C metadata=98e89df75c36be48 -C extra-filename=-98e89df75c36be48 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling typify-macro v0.0.13 (https://github.com/oxidecomputer/typify#92bfed8b)
     Running `rustc --crate-name typify_macro --edition=2021 /home/build/.cargo/git/checkouts/typify-288d5a84bbbe6a46/92bfed8/typify-macro/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=f2da351574d5abd0 -C extra-filename=-f2da351574d5abd0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern serde_tokenstream=/work/oxidecomputer/crucible/target/debug/deps/libserde_tokenstream-2a83027836bfd0fb.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern typify_impl=/work/oxidecomputer/crucible/target/debug/deps/libtypify_impl-5385296f2ea50467.rlib --extern proc_macro --cap-lints allow`
   Compiling parse-display v0.7.0
     Running `rustc --crate-name parse_display --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/parse-display-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="once_cell"' --cfg 'feature="regex"' --cfg 'feature="std"' -C metadata=34a1a5d52375b70b -C extra-filename=-34a1a5d52375b70b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern parse_display_derive=/work/oxidecomputer/crucible/target/debug/deps/libparse_display_derive-95e2e5d955c96bcf.so --extern regex=/work/oxidecomputer/crucible/target/debug/deps/libregex-f9e3a4eb3da387ce.rmeta --cap-lints allow`
   Compiling api_identity v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name api_identity --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/api_identity/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=90d45ecc06c8f773 -C extra-filename=-90d45ecc06c8f773 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling base64ct v1.6.0
     Running `rustc --crate-name base64ct --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/base64ct-1.6.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="std"' -C metadata=5aa1f74fadba3334 -C extra-filename=-5aa1f74fadba3334 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling bincode v1.3.3
     Running `rustc --crate-name bincode /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bincode-1.3.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bcb925e8faac86cd -C extra-filename=-bcb925e8faac86cd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
   Compiling password-hash v0.5.0
     Running `rustc --crate-name password_hash --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/password-hash-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="rand_core"' --cfg 'feature="std"' -C metadata=6571709e5cce3ff3 -C extra-filename=-6571709e5cce3ff3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64ct=/work/oxidecomputer/crucible/target/debug/deps/libbase64ct-5aa1f74fadba3334.rmeta --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-e2870cc0e01c33c9.rmeta --extern subtle=/work/oxidecomputer/crucible/target/debug/deps/libsubtle-ee0c8d46ce57336b.rmeta --cap-lints allow`
   Compiling blake2 v0.10.6
     Running `rustc --crate-name blake2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/blake2-0.10.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=39c009971635667f -C extra-filename=-39c009971635667f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern digest=/work/oxidecomputer/crucible/target/debug/deps/libdigest-d0d34473efe105d1.rmeta --cap-lints allow`
   Compiling inout v0.1.3
     Running `rustc --crate-name inout --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/inout-0.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=63af7c577909eb44 -C extra-filename=-63af7c577909eb44 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern generic_array=/work/oxidecomputer/crucible/target/debug/deps/libgeneric_array-bc31b01978a602e7.rmeta --cap-lints allow`
   Compiling universal-hash v0.5.0
     Running `rustc --crate-name universal_hash --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/universal-hash-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4588bd931f24d922 -C extra-filename=-4588bd931f24d922 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crypto_common=/work/oxidecomputer/crucible/target/debug/deps/libcrypto_common-0953bfc5dcef84b9.rmeta --extern subtle=/work/oxidecomputer/crucible/target/debug/deps/libsubtle-ee0c8d46ce57336b.rmeta --cap-lints allow`
   Compiling cipher v0.4.3
     Running `rustc --crate-name cipher --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/cipher-0.4.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fe5b83e4e2b73d72 -C extra-filename=-fe5b83e4e2b73d72 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crypto_common=/work/oxidecomputer/crucible/target/debug/deps/libcrypto_common-0953bfc5dcef84b9.rmeta --extern inout=/work/oxidecomputer/crucible/target/debug/deps/libinout-63af7c577909eb44.rmeta --cap-lints allow`
   Compiling argon2 v0.5.0
     Running `rustc --crate-name argon2 --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/argon2-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="password-hash"' --cfg 'feature="rand"' --cfg 'feature="std"' -C metadata=8f03ab3a8d12bcba -C extra-filename=-8f03ab3a8d12bcba --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64ct=/work/oxidecomputer/crucible/target/debug/deps/libbase64ct-5aa1f74fadba3334.rmeta --extern blake2=/work/oxidecomputer/crucible/target/debug/deps/libblake2-39c009971635667f.rmeta --extern password_hash=/work/oxidecomputer/crucible/target/debug/deps/libpassword_hash-6571709e5cce3ff3.rmeta --cap-lints allow`
   Compiling oximeter-macro-impl v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name oximeter_macro_impl --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/oximeter/oximeter-macro-impl/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=e4cc949eda20c416 -C extra-filename=-e4cc949eda20c416 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling num_enum v0.7.0
     Running `rustc --crate-name num_enum --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num_enum-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=9cd7a6d9dcf1dd5a -C extra-filename=-9cd7a6d9dcf1dd5a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_enum_derive=/work/oxidecomputer/crucible/target/debug/deps/libnum_enum_derive-2bef02e01d5b06d3.so --cap-lints allow`
   Compiling opaque-debug v0.3.0
     Running `rustc --crate-name opaque_debug --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opaque-debug-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bdee85e4e8a367bc -C extra-filename=-bdee85e4e8a367bc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling aes v0.8.2
     Running `rustc --crate-name aes --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/aes-0.8.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e0592e8e1d03d860 -C extra-filename=-e0592e8e1d03d860 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern cipher=/work/oxidecomputer/crucible/target/debug/deps/libcipher-fe5b83e4e2b73d72.rmeta --extern cpufeatures=/work/oxidecomputer/crucible/target/debug/deps/libcpufeatures-6e25ef20a8fa218f.rmeta --cap-lints allow`
   Compiling omicron-passwords v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name omicron_passwords --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/passwords/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ac6e3a602e6ad041 -C extra-filename=-ac6e3a602e6ad041 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern argon2=/work/oxidecomputer/crucible/target/debug/deps/libargon2-8f03ab3a8d12bcba.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_with=/work/oxidecomputer/crucible/target/debug/deps/libserde_with-4f9ddd30b380d6cf.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --cap-lints allow`
   Compiling polyval v0.6.0
     Running `rustc --crate-name polyval --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/polyval-0.6.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e4b47d7c1f31f998 -C extra-filename=-e4b47d7c1f31f998 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern cpufeatures=/work/oxidecomputer/crucible/target/debug/deps/libcpufeatures-6e25ef20a8fa218f.rmeta --extern opaque_debug=/work/oxidecomputer/crucible/target/debug/deps/libopaque_debug-bdee85e4e8a367bc.rmeta --extern universal_hash=/work/oxidecomputer/crucible/target/debug/deps/libuniversal_hash-4588bd931f24d922.rmeta --cap-lints allow`
   Compiling ctr v0.9.2
     Running `rustc --crate-name ctr --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ctr-0.9.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=583e100d27882194 -C extra-filename=-583e100d27882194 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cipher=/work/oxidecomputer/crucible/target/debug/deps/libcipher-fe5b83e4e2b73d72.rmeta --cap-lints allow`
   Compiling crucible-client-types v0.1.0 (/work/oxidecomputer/crucible/crucible-client-types)
     Running `rustc --crate-name crucible_client_types --edition=2021 crucible-client-types/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bd54c4335d2370bd -C extra-filename=-bd54c4335d2370bd --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta`
   Compiling aead v0.5.1
     Running `rustc --crate-name aead --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/aead-0.5.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="getrandom"' --cfg 'feature="rand_core"' -C metadata=f30da292094eb963 -C extra-filename=-f30da292094eb963 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crypto_common=/work/oxidecomputer/crucible/target/debug/deps/libcrypto_common-0953bfc5dcef84b9.rmeta --extern generic_array=/work/oxidecomputer/crucible/target/debug/deps/libgeneric_array-bc31b01978a602e7.rmeta --cap-lints allow`
   Compiling crucible v0.0.1 (/work/oxidecomputer/crucible/upstairs)
     Running `rustc --crate-name build_script_build --edition=2021 upstairs/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=1852afdb7baa90de -C extra-filename=-1852afdb7baa90de --out-dir /work/oxidecomputer/crucible/target/debug/build/crucible-1852afdb7baa90de -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern version_check=/work/oxidecomputer/crucible/target/debug/deps/libversion_check-23142ed5bf6178a0.rlib`
   Compiling zeroize v1.3.0
     Running `rustc --crate-name zeroize --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/zeroize-1.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=8e99cf4813483d58 -C extra-filename=-8e99cf4813483d58 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling itertools v0.11.0
     Running `rustc --crate-name itertools --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/itertools-0.11.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="use_alloc"' --cfg 'feature="use_std"' -C metadata=b06e69badd72e55c -C extra-filename=-b06e69badd72e55c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern either=/work/oxidecomputer/crucible/target/debug/deps/libeither-e075e5427abad613.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/crucible-1852afdb7baa90de/build-script-build`
   Compiling aes-gcm-siv v0.11.1
     Running `rustc --crate-name aes_gcm_siv --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/aes-gcm-siv-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="aes"' --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="getrandom"' -C metadata=21495b616a07c9a4 -C extra-filename=-21495b616a07c9a4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern aead=/work/oxidecomputer/crucible/target/debug/deps/libaead-f30da292094eb963.rmeta --extern aes=/work/oxidecomputer/crucible/target/debug/deps/libaes-e0592e8e1d03d860.rmeta --extern cipher=/work/oxidecomputer/crucible/target/debug/deps/libcipher-fe5b83e4e2b73d72.rmeta --extern ctr=/work/oxidecomputer/crucible/target/debug/deps/libctr-583e100d27882194.rmeta --extern polyval=/work/oxidecomputer/crucible/target/debug/deps/libpolyval-e4b47d7c1f31f998.rmeta --extern subtle=/work/oxidecomputer/crucible/target/debug/deps/libsubtle-ee0c8d46ce57336b.rmeta --extern zeroize=/work/oxidecomputer/crucible/target/debug/deps/libzeroize-8e99cf4813483d58.rmeta --cap-lints allow`
   Compiling async-recursion v1.0.5
     Running `rustc --crate-name async_recursion --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-recursion-1.0.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=ce9499495a1cb858 -C extra-filename=-ce9499495a1cb858 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling ringbuffer v0.14.2
     Running `rustc --crate-name ringbuffer --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ringbuffer-0.14.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' -C metadata=e91d75e4694e6351 -C extra-filename=-e91d75e4694e6351 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling unicode-segmentation v1.10.0
     Running `rustc --crate-name unicode_segmentation --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicode-segmentation-1.10.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=06176721b7b95955 -C extra-filename=-06176721b7b95955 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling console v0.15.5
     Running `rustc --crate-name console --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/console-0.15.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="ansi-parsing"' --cfg 'feature="default"' --cfg 'feature="unicode-width"' -C metadata=4236472a6e29ce0a -C extra-filename=-4236472a6e29ce0a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern unicode_width=/work/oxidecomputer/crucible/target/debug/deps/libunicode_width-87c5262a4c4bb0e9.rmeta --cap-lints allow`
   Compiling opentelemetry_api v0.18.0
     Running `rustc --crate-name opentelemetry_api --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry_api-0.18.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="fnv"' --cfg 'feature="metrics"' --cfg 'feature="pin-project-lite"' --cfg 'feature="trace"' -C metadata=54be0f0e21b05b9c -C extra-filename=-54be0f0e21b05b9c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --cap-lints allow`
   Compiling dashmap v5.2.0
     Running `rustc --crate-name dashmap --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dashmap-5.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="send_guard"' -C metadata=e6d3c88fce252f4a -C extra-filename=-e6d3c88fce252f4a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern num_cpus=/work/oxidecomputer/crucible/target/debug/deps/libnum_cpus-67a451bebfcc5086.rmeta --extern parking_lot=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot-970d5c0acece447c.rmeta --cap-lints allow`
   Compiling unicase v2.6.0
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicase-2.6.0/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=a2e7dddab165438a -C extra-filename=-a2e7dddab165438a --out-dir /work/oxidecomputer/crucible/target/debug/build/unicase-a2e7dddab165438a -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern version_check=/work/oxidecomputer/crucible/target/debug/deps/libversion_check-23142ed5bf6178a0.rlib --cap-lints allow`
   Compiling newline-converter v0.3.0
     Running `rustc --crate-name newline_converter --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/newline-converter-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=54456d7e7e0ff19b -C extra-filename=-54456d7e7e0ff19b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern unicode_segmentation=/work/oxidecomputer/crucible/target/debug/deps/libunicode_segmentation-06176721b7b95955.rmeta --cap-lints allow`
   Compiling num-derive v0.4.0
     Running `rustc --crate-name num_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/num-derive-0.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=814c8a0a0a713cba -C extra-filename=-814c8a0a0a713cba --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/unicase-a2e7dddab165438a/build-script-build`
   Compiling similar v2.2.1
     Running `rustc --crate-name similar --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/similar-2.2.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="text"' -C metadata=c5a411995c7e1b53 -C extra-filename=-c5a411995c7e1b53 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling typify v0.0.13 (https://github.com/oxidecomputer/typify#92bfed8b)
     Running `rustc --crate-name typify --edition=2021 /home/build/.cargo/git/checkouts/typify-288d5a84bbbe6a46/92bfed8/typify/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="macro"' --cfg 'feature="typify-macro"' -C metadata=60072fbcea665837 -C extra-filename=-60072fbcea665837 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern typify_impl=/work/oxidecomputer/crucible/target/debug/deps/libtypify_impl-5385296f2ea50467.rmeta --extern typify_macro=/work/oxidecomputer/crucible/target/debug/deps/libtypify_macro-f2da351574d5abd0.so --cap-lints allow`
   Compiling opentelemetry_sdk v0.18.0
     Running `rustc --crate-name opentelemetry_sdk --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry_sdk-0.18.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="async-trait"' --cfg 'feature="crossbeam-channel"' --cfg 'feature="dashmap"' --cfg 'feature="default"' --cfg 'feature="fnv"' --cfg 'feature="metrics"' --cfg 'feature="percent-encoding"' --cfg 'feature="rand"' --cfg 'feature="trace"' -C metadata=f1be95b27d4b99f6 -C extra-filename=-f1be95b27d4b99f6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern crossbeam_channel=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_channel-19ff6fa486e51c63.rmeta --extern dashmap=/work/oxidecomputer/crucible/target/debug/deps/libdashmap-e6d3c88fce252f4a.rmeta --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_executor=/work/oxidecomputer/crucible/target/debug/deps/libfutures_executor-61f53162107ffb32.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern opentelemetry_api=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_api-54be0f0e21b05b9c.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --cap-lints allow`
   Compiling convert_case v0.5.0
     Running `rustc --crate-name convert_case --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/convert_case-0.5.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=1b3c1b1b6d16787f -C extra-filename=-1b3c1b1b6d16787f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling progenitor-impl v0.3.0 (https://github.com/oxidecomputer/progenitor?branch=main#76716eea)
     Running `rustc --crate-name progenitor_impl --edition=2021 /home/build/.cargo/git/checkouts/progenitor-639bd64206ac5e43/76716ee/progenitor-impl/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4eeb9d4c0f08c454 -C extra-filename=-4eeb9d4c0f08c454 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern getopts=/work/oxidecomputer/crucible/target/debug/deps/libgetopts-93a8419d37acce69.rmeta --extern heck=/work/oxidecomputer/crucible/target/debug/deps/libheck-29af75c938b110f7.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern regex=/work/oxidecomputer/crucible/target/debug/deps/libregex-f9e3a4eb3da387ce.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern typify=/work/oxidecomputer/crucible/target/debug/deps/libtypify-60072fbcea665837.rmeta --extern unicode_ident=/work/oxidecomputer/crucible/target/debug/deps/libunicode_ident-81e7752fff89e70f.rmeta --cap-lints allow`
   Compiling openapi-lint v0.1.0 (https://github.com/oxidecomputer/openapi-lint?branch=main#9e3ada82)
     Running `rustc --crate-name openapi_lint --edition=2018 /home/build/.cargo/git/checkouts/openapi-lint-42a90d71bc44c8c6/9e3ada8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=848bb99097a9a843 -C extra-filename=-848bb99097a9a843 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern convert_case=/work/oxidecomputer/crucible/target/debug/deps/libconvert_case-1b3c1b1b6d16787f.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rmeta --cap-lints allow`
   Compiling expectorate v1.0.7
     Running `rustc --crate-name expectorate --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/expectorate-1.0.7/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4e45b262baa473cc -C extra-filename=-4e45b262baa473cc --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern console=/work/oxidecomputer/crucible/target/debug/deps/libconsole-4236472a6e29ce0a.rmeta --extern newline_converter=/work/oxidecomputer/crucible/target/debug/deps/libnewline_converter-54456d7e7e0ff19b.rmeta --extern similar=/work/oxidecomputer/crucible/target/debug/deps/libsimilar-c5a411995c7e1b53.rmeta --cap-lints allow`
     Running `rustc --crate-name unicase /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unicase-2.6.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=60e4f37ecb23afe1 -C extra-filename=-60e4f37ecb23afe1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg __unicase__iter_cmp --cfg __unicase__default_hasher --cfg __unicase__const_fns --cfg __unicase__core_and_alloc`
   Compiling opentelemetry v0.18.0
     Running `rustc --crate-name opentelemetry --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry-0.18.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="metrics"' --cfg 'feature="trace"' -C metadata=f41372bdd269a545 -C extra-filename=-f41372bdd269a545 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern opentelemetry_api=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_api-54be0f0e21b05b9c.rmeta --extern opentelemetry_sdk=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_sdk-f1be95b27d4b99f6.rmeta --cap-lints allow`
   Compiling overload v0.1.1
     Running `rustc --crate-name overload --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/overload-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c7048b1d029106d7 -C extra-filename=-c7048b1d029106d7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling urlencoding v2.1.2
   Compiling nu-ansi-term v0.46.0
     Running `rustc --crate-name urlencoding --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/urlencoding-2.1.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=76c058d20df73e40 -C extra-filename=-76c058d20df73e40 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name nu_ansi_term --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/nu-ansi-term-0.46.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=91641c4d652bb213 -C extra-filename=-91641c4d652bb213 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern overload=/work/oxidecomputer/crucible/target/debug/deps/liboverload-c7048b1d029106d7.rmeta --cap-lints allow`
   Compiling crucible-smf v0.0.0 (/work/oxidecomputer/crucible/smf)
     Running `rustc --crate-name crucible_smf --edition=2021 smf/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e1a82c6f17385dc6 -C extra-filename=-e1a82c6f17385dc6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern num_derive=/work/oxidecomputer/crucible/target/debug/deps/libnum_derive-814c8a0a0a713cba.so --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta`
   Compiling opentelemetry_api v0.20.0
   Compiling mime_guess v2.0.4
     Running `rustc --crate-name opentelemetry_api --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry_api-0.20.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="pin-project-lite"' --cfg 'feature="trace"' -C metadata=04e099a48329a859 -C extra-filename=-04e099a48329a859 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern urlencoding=/work/oxidecomputer/crucible/target/debug/deps/liburlencoding-76c058d20df73e40.rmeta --cap-lints allow`
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mime_guess-2.0.4/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="rev-mappings"' -C metadata=dbd0371f2a220632 -C extra-filename=-dbd0371f2a220632 --out-dir /work/oxidecomputer/crucible/target/debug/build/mime_guess-dbd0371f2a220632 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern unicase=/work/oxidecomputer/crucible/target/debug/deps/libunicase-60e4f37ecb23afe1.rlib --cap-lints allow`
   Compiling tracing-log v0.1.3
     Running `rustc --crate-name tracing_log --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-log-0.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="log-tracer"' --cfg 'feature="std"' -C metadata=d77683210f5d223d -C extra-filename=-d77683210f5d223d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern tracing_core=/work/oxidecomputer/crucible/target/debug/deps/libtracing_core-adac59f754126e83.rmeta --cap-lints allow`
   Compiling sharded-slab v0.1.4
     Running `rustc --crate-name sharded_slab --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/sharded-slab-0.1.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=9ba6a384880b5495 -C extra-filename=-9ba6a384880b5495 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --cap-lints allow`
   Compiling ordered-float v1.1.1
     Running `rustc --crate-name ordered_float /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ordered-float-1.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=b7d47a63e5afa582 -C extra-filename=-b7d47a63e5afa582 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow`
   Compiling ordered-float v3.7.0
     Running `rustc --crate-name ordered_float --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ordered-float-3.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=c435708391037060 -C extra-filename=-c435708391037060 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --cap-lints allow`
   Compiling threadpool v1.8.1
     Running `rustc --crate-name threadpool /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/threadpool-1.8.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=e4e99e14eaa48b3e -C extra-filename=-e4e99e14eaa48b3e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern num_cpus=/work/oxidecomputer/crucible/target/debug/deps/libnum_cpus-67a451bebfcc5086.rmeta --cap-lints allow`
   Compiling signal-hook v0.3.17
   Compiling integer-encoding v3.0.3
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/signal-hook-0.3.17/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="channel"' --cfg 'feature="default"' --cfg 'feature="iterator"' -C metadata=dc2729fde466a024 -C extra-filename=-dc2729fde466a024 --out-dir /work/oxidecomputer/crucible/target/debug/build/signal-hook-dc2729fde466a024 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `rustc --crate-name integer_encoding --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/integer-encoding-3.0.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=d6f198d882d03cc4 -C extra-filename=-d6f198d882d03cc4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/signal-hook-dc2729fde466a024/build-script-build`
   Compiling opentelemetry_sdk v0.20.0
     Running `rustc --crate-name opentelemetry_sdk --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry_sdk-0.20.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="async-trait"' --cfg 'feature="crossbeam-channel"' --cfg 'feature="default"' --cfg 'feature="percent-encoding"' --cfg 'feature="rand"' --cfg 'feature="trace"' -C metadata=6fef3b11a1793dc3 -C extra-filename=-6fef3b11a1793dc3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern crossbeam_channel=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_channel-19ff6fa486e51c63.rmeta --extern futures_channel=/work/oxidecomputer/crucible/target/debug/deps/libfutures_channel-34a7a018f58dc5a7.rmeta --extern futures_executor=/work/oxidecomputer/crucible/target/debug/deps/libfutures_executor-61f53162107ffb32.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern opentelemetry_api=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_api-04e099a48329a859.rmeta --extern ordered_float=/work/oxidecomputer/crucible/target/debug/deps/libordered_float-c435708391037060.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --cap-lints allow`
   Compiling thrift v0.16.0
     Running `rustc --crate-name thrift --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/thrift-0.16.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="log"' --cfg 'feature="server"' --cfg 'feature="threadpool"' -C metadata=1fac1fb5888ad036 -C extra-filename=-1fac1fb5888ad036 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern integer_encoding=/work/oxidecomputer/crucible/target/debug/deps/libinteger_encoding-d6f198d882d03cc4.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern ordered_float=/work/oxidecomputer/crucible/target/debug/deps/libordered_float-b7d47a63e5afa582.rmeta --extern threadpool=/work/oxidecomputer/crucible/target/debug/deps/libthreadpool-e4e99e14eaa48b3e.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/mime_guess-dbd0371f2a220632/build-script-build`
   Compiling opentelemetry-semantic-conventions v0.10.0
     Running `rustc --crate-name opentelemetry_semantic_conventions --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry-semantic-conventions-0.10.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=18113e5418aec521 -C extra-filename=-18113e5418aec521 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-f41372bdd269a545.rmeta --cap-lints allow`
   Compiling tracing-subscriber v0.3.17
     Running `rustc --crate-name tracing_subscriber --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-subscriber-0.3.17/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="ansi"' --cfg 'feature="default"' --cfg 'feature="fmt"' --cfg 'feature="nu-ansi-term"' --cfg 'feature="registry"' --cfg 'feature="sharded-slab"' --cfg 'feature="smallvec"' --cfg 'feature="std"' --cfg 'feature="thread_local"' --cfg 'feature="tracing-log"' -C metadata=e39dae5ba339bc78 -C extra-filename=-e39dae5ba339bc78 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern nu_ansi_term=/work/oxidecomputer/crucible/target/debug/deps/libnu_ansi_term-91641c4d652bb213.rmeta --extern sharded_slab=/work/oxidecomputer/crucible/target/debug/deps/libsharded_slab-9ba6a384880b5495.rmeta --extern smallvec=/work/oxidecomputer/crucible/target/debug/deps/libsmallvec-397f26bd8c84e528.rmeta --extern thread_local=/work/oxidecomputer/crucible/target/debug/deps/libthread_local-771d112d9219dc9f.rmeta --extern tracing_core=/work/oxidecomputer/crucible/target/debug/deps/libtracing_core-adac59f754126e83.rmeta --extern tracing_log=/work/oxidecomputer/crucible/target/debug/deps/libtracing_log-d77683210f5d223d.rmeta --cap-lints allow`
   Compiling phf_shared v0.11.1
     Running `rustc --crate-name phf_shared --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf_shared-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=f4c611c5b95ef91c -C extra-filename=-f4c611c5b95ef91c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern siphasher=/work/oxidecomputer/crucible/target/debug/deps/libsiphasher-2eb27360f66646f3.rmeta --cap-lints allow`
   Compiling memoffset v0.6.5
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/memoffset-0.6.5/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' -C metadata=98b281bc850662f7 -C extra-filename=-98b281bc850662f7 --out-dir /work/oxidecomputer/crucible/target/debug/build/memoffset-98b281bc850662f7 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
     Running `rustc --crate-name mime_guess /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/mime_guess-2.0.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="rev-mappings"' -C metadata=66974d6c31968dc2 -C extra-filename=-66974d6c31968dc2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern mime=/work/oxidecomputer/crucible/target/debug/deps/libmime-ac14a9115eddd3c2.rmeta --extern unicase=/work/oxidecomputer/crucible/target/debug/deps/libunicase-60e4f37ecb23afe1.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/memoffset-98b281bc850662f7/build-script-build`
   Compiling opentelemetry v0.20.0
     Running `rustc --crate-name opentelemetry --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry-0.20.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="trace"' -C metadata=5524fd7817ad57fb -C extra-filename=-5524fd7817ad57fb --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern opentelemetry_api=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_api-04e099a48329a859.rmeta --extern opentelemetry_sdk=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_sdk-6fef3b11a1793dc3.rmeta --cap-lints allow`
   Compiling opentelemetry-jaeger v0.17.0
     Running `rustc --crate-name opentelemetry_jaeger --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/opentelemetry-jaeger-0.17.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=9ebf47742e5e063f -C extra-filename=-9ebf47742e5e063f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern futures_executor=/work/oxidecomputer/crucible/target/debug/deps/libfutures_executor-61f53162107ffb32.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-f41372bdd269a545.rmeta --extern opentelemetry_semantic_conventions=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_semantic_conventions-18113e5418aec521.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern thrift=/work/oxidecomputer/crucible/target/debug/deps/libthrift-1fac1fb5888ad036.rmeta --cap-lints allow`
     Running `rustc --crate-name signal_hook --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/signal-hook-0.3.17/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="channel"' --cfg 'feature="default"' --cfg 'feature="iterator"' -C metadata=ae206b38743b6815 -C extra-filename=-ae206b38743b6815 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern signal_hook_registry=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_registry-4d955479f235827e.rmeta --cap-lints allow`
   Compiling crossbeam-epoch v0.9.8
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossbeam-epoch-0.9.8/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="alloc"' --cfg 'feature="lazy_static"' --cfg 'feature="std"' -C metadata=d874a78423ef46ce -C extra-filename=-d874a78423ef46ce --out-dir /work/oxidecomputer/crucible/target/debug/build/crossbeam-epoch-d874a78423ef46ce -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
   Compiling crucible-downstairs v0.0.1 (/work/oxidecomputer/crucible/downstairs)
     Running `rustc --crate-name build_script_build --edition=2021 downstairs/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' -C metadata=66ed07e0d9dc333b -C extra-filename=-66ed07e0d9dc333b --out-dir /work/oxidecomputer/crucible/target/debug/build/crucible-downstairs-66ed07e0d9dc333b -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern version_check=/work/oxidecomputer/crucible/target/debug/deps/libversion_check-23142ed5bf6178a0.rlib`
   Compiling tracing-opentelemetry v0.18.0
     Running `rustc --crate-name tracing_opentelemetry --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-opentelemetry-0.18.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="metrics"' --cfg 'feature="tracing-log"' -C metadata=8f8cfc1900c3a663 -C extra-filename=-8f8cfc1900c3a663 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-f41372bdd269a545.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --extern tracing_core=/work/oxidecomputer/crucible/target/debug/deps/libtracing_core-adac59f754126e83.rmeta --extern tracing_log=/work/oxidecomputer/crucible/target/debug/deps/libtracing_log-d77683210f5d223d.rmeta --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rmeta --cap-lints allow`
   Compiling http-range v0.1.5
     Running `rustc --crate-name http_range --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/http-range-0.1.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=4885ae91fe1ea75f -C extra-filename=-4885ae91fe1ea75f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/crucible-downstairs-66ed07e0d9dc333b/build-script-build`
   Compiling hyper-staticfile v0.9.4
     Running `rustc --crate-name hyper_staticfile --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hyper-staticfile-0.9.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=559b4389ef952563 -C extra-filename=-559b4389ef952563 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern http_range=/work/oxidecomputer/crucible/target/debug/deps/libhttp_range-4885ae91fe1ea75f.rmeta --extern httpdate=/work/oxidecomputer/crucible/target/debug/deps/libhttpdate-e3ef82a990113a54.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern mime_guess=/work/oxidecomputer/crucible/target/debug/deps/libmime_guess-66974d6c31968dc2.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern url=/work/oxidecomputer/crucible/target/debug/deps/liburl-ff56943ab9066fdc.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/crossbeam-epoch-d874a78423ef46ce/build-script-build`
     Running `rustc --crate-name memoffset /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/memoffset-0.6.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=508ecbf92309dc73 -C extra-filename=-508ecbf92309dc73 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg tuple_ty --cfg allow_clippy --cfg maybe_uninit --cfg doctests --cfg raw_ref_macros`
   Compiling phf_generator v0.11.1
     Running `rustc --crate-name phf_generator --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf_generator-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=41d436cb237437bf -C extra-filename=-41d436cb237437bf --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern phf_shared=/work/oxidecomputer/crucible/target/debug/deps/libphf_shared-f4c611c5b95ef91c.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --cap-lints allow`
   Compiling io-lifetimes v0.5.3
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/io-lifetimes-0.5.3/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=eca1b29fac9a0fbe -C extra-filename=-eca1b29fac9a0fbe --out-dir /work/oxidecomputer/crucible/target/debug/build/io-lifetimes-eca1b29fac9a0fbe -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling rayon-core v1.9.2
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rayon-core-1.9.2/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=b3156e65197adcc4 -C extra-filename=-b3156e65197adcc4 --out-dir /work/oxidecomputer/crucible/target/debug/build/rayon-core-b3156e65197adcc4 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/rayon-core-b3156e65197adcc4/build-script-build`
   Compiling phf_codegen v0.11.1
     Running `rustc --crate-name phf_codegen --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf_codegen-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=f8063558873b33a1 -C extra-filename=-f8063558873b33a1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern phf_generator=/work/oxidecomputer/crucible/target/debug/deps/libphf_generator-41d436cb237437bf.rmeta --extern phf_shared=/work/oxidecomputer/crucible/target/debug/deps/libphf_shared-f4c611c5b95ef91c.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/io-lifetimes-eca1b29fac9a0fbe/build-script-build`
     Running `rustc --crate-name crossbeam_epoch --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossbeam-epoch-0.9.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="lazy_static"' --cfg 'feature="std"' -C metadata=44fc1cc817fcdd23 -C extra-filename=-44fc1cc817fcdd23 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern crossbeam_utils=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_utils-a1fb255bfa31483a.rmeta --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern memoffset=/work/oxidecomputer/crucible/target/debug/deps/libmemoffset-508ecbf92309dc73.rmeta --extern scopeguard=/work/oxidecomputer/crucible/target/debug/deps/libscopeguard-00b7ece4eb7b8e7e.rmeta --cap-lints allow --cfg crossbeam_const_fn_trait_bound`
   Compiling rustix v0.33.4
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustix-0.33.4/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="io-lifetimes"' --cfg 'feature="std"' -C metadata=4d7937b0898004d1 -C extra-filename=-4d7937b0898004d1 --out-dir /work/oxidecomputer/crucible/target/debug/build/rustix-4d7937b0898004d1 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling crc32fast v1.3.2
     Running `rustc --crate-name build_script_build /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crc32fast-1.3.2/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=5d7fac4f45a97417 -C extra-filename=-5d7fac4f45a97417 --out-dir /work/oxidecomputer/crucible/target/debug/build/crc32fast-5d7fac4f45a97417 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling terminfo v0.8.0
     Running `rustc --crate-name build_script_build --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/terminfo-0.8.0/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=bed6d7dcce5eac66 -C extra-filename=-bed6d7dcce5eac66 --out-dir /work/oxidecomputer/crucible/target/debug/build/terminfo-bed6d7dcce5eac66 -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern phf_codegen=/work/oxidecomputer/crucible/target/debug/deps/libphf_codegen-f8063558873b33a1.rlib --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/rustix-4d7937b0898004d1/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/crc32fast-5d7fac4f45a97417/build-script-build`
   Compiling crossbeam-deque v0.8.1
     Running `rustc --crate-name crossbeam_deque --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossbeam-deque-0.8.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="crossbeam-epoch"' --cfg 'feature="crossbeam-utils"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=46df68c0a92e69f5 -C extra-filename=-46df68c0a92e69f5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern crossbeam_epoch=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_epoch-44fc1cc817fcdd23.rmeta --extern crossbeam_utils=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_utils-a1fb255bfa31483a.rmeta --cap-lints allow`
     Running `rustc --crate-name io_lifetimes --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/io-lifetimes-0.5.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=59f01ff60b1f438f -C extra-filename=-59f01ff60b1f438f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling signal-hook-mio v0.2.3
     Running `rustc --crate-name signal_hook_mio --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/signal-hook-mio-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="mio-0_8"' --cfg 'feature="support-v0_8"' -C metadata=b78bc232ff08be19 -C extra-filename=-b78bc232ff08be19 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern mio_0_8=/work/oxidecomputer/crucible/target/debug/deps/libmio-27a8136cf12de2bb.rmeta --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rmeta --cap-lints allow`
   Compiling structmeta-derive v0.2.0
     Running `rustc --crate-name structmeta_derive --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/structmeta-derive-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=6868cfc37bbab5f5 -C extra-filename=-6868cfc37bbab5f5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling vte_generate_state_changes v0.1.1
     Running `rustc --crate-name vte_generate_state_changes --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/vte_generate_state_changes-0.1.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=52ea4d6c5be5fc21 -C extra-filename=-52ea4d6c5be5fc21 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern proc_macro --cap-lints allow`
   Compiling subprocess v0.2.9
     Running `rustc --crate-name subprocess --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/subprocess-0.2.9/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0acfc5c9b903588a -C extra-filename=-0acfc5c9b903588a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling dirs-sys v0.3.7
     Running `rustc --crate-name dirs_sys /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dirs-sys-0.3.7/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=15239ab4fd89ac89 -C extra-filename=-15239ab4fd89ac89 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling wait-timeout v0.2.0
     Running `rustc --crate-name wait_timeout /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/wait-timeout-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=509e2becba2710d2 -C extra-filename=-509e2becba2710d2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling rayon v1.5.2
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rayon-1.5.2/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=03d683edffe821df -C extra-filename=-03d683edffe821df --out-dir /work/oxidecomputer/crucible/target/debug/build/rayon-03d683edffe821df -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern autocfg=/work/oxidecomputer/crucible/target/debug/deps/libautocfg-b1cec8cc882d2fdd.rlib --cap-lints allow`
   Compiling portable-atomic v1.4.1
     Running `rustc --crate-name build_script_build --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/portable-atomic-1.4.1/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no --cfg 'feature="default"' --cfg 'feature="fallback"' -C metadata=89385d5be32ea8bc -C extra-filename=-89385d5be32ea8bc --out-dir /work/oxidecomputer/crucible/target/debug/build/portable-atomic-89385d5be32ea8bc -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling progenitor-macro v0.3.0 (https://github.com/oxidecomputer/progenitor?branch=main#76716eea)
     Running `rustc --crate-name progenitor_macro --edition=2021 /home/build/.cargo/git/checkouts/progenitor-639bd64206ac5e43/76716ee/progenitor-macro/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=4c13a8353939c841 -C extra-filename=-4c13a8353939c841 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern progenitor_impl=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor_impl-4eeb9d4c0f08c454.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern serde_tokenstream=/work/oxidecomputer/crucible/target/debug/deps/libserde_tokenstream-2a83027836bfd0fb.rlib --extern serde_yaml=/work/oxidecomputer/crucible/target/debug/deps/libserde_yaml-c7db3f53bbf8134b.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling bit-vec v0.6.3
     Running `rustc --crate-name bit_vec /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bit-vec-0.6.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=35546b1439b435ea -C extra-filename=-35546b1439b435ea --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling adler v1.0.2
     Running `rustc --crate-name adler /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/adler-1.0.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=de7aee9f30a32373 -C extra-filename=-de7aee9f30a32373 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling regex-automata v0.1.10
     Running `rustc --crate-name regex_automata /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/regex-automata-0.1.10/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=054d4c6c4feb4142 -C extra-filename=-054d4c6c4feb4142 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling minimal-lexical v0.2.1
     Running `rustc --crate-name minimal_lexical --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/minimal-lexical-0.2.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="std"' -C metadata=fec5e2c45ccf032b -C extra-filename=-fec5e2c45ccf032b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling quick-error v1.2.3
     Running `rustc --crate-name quick_error /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/quick-error-1.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c8c4727ecc8bb914 -C extra-filename=-c8c4727ecc8bb914 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling bit-set v0.5.3
     Running `rustc --crate-name bit_set /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bit-set-0.5.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=ec12e43f0b47413c -C extra-filename=-ec12e43f0b47413c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bit_vec=/work/oxidecomputer/crucible/target/debug/deps/libbit_vec-35546b1439b435ea.rmeta --cap-lints allow`
   Compiling bstr v0.2.17
     Running `rustc --crate-name bstr --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/bstr-0.2.17/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="lazy_static"' --cfg 'feature="regex-automata"' --cfg 'feature="std"' --cfg 'feature="unicode"' -C metadata=f0785d87dddebfcd -C extra-filename=-f0785d87dddebfcd --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --extern regex_automata=/work/oxidecomputer/crucible/target/debug/deps/libregex_automata-054d4c6c4feb4142.rmeta --cap-lints allow`
   Compiling nom v7.1.3
     Running `rustc --crate-name nom --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/nom-7.1.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="std"' -C metadata=7d7f045245d4ae57 -C extra-filename=-7d7f045245d4ae57 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --extern minimal_lexical=/work/oxidecomputer/crucible/target/debug/deps/libminimal_lexical-fec5e2c45ccf032b.rmeta --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/portable-atomic-89385d5be32ea8bc/build-script-build`
   Compiling rusty-fork v0.3.0
     Running `rustc --crate-name rusty_fork --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rusty-fork-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="timeout"' --cfg 'feature="wait-timeout"' -C metadata=20bbf553eecf6baa -C extra-filename=-20bbf553eecf6baa --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern quick_error=/work/oxidecomputer/crucible/target/debug/deps/libquick_error-c8c4727ecc8bb914.rmeta --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rmeta --extern wait_timeout=/work/oxidecomputer/crucible/target/debug/deps/libwait_timeout-509e2becba2710d2.rmeta --cap-lints allow`
   Compiling miniz_oxide v0.6.2
     Running `rustc --crate-name miniz_oxide --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/miniz_oxide-0.6.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="with-alloc"' -C metadata=f7bade0e2b1a9da4 -C extra-filename=-f7bade0e2b1a9da4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern adler=/work/oxidecomputer/crucible/target/debug/deps/libadler-de7aee9f30a32373.rmeta --cap-lints allow`
   Compiling vte v0.11.1
     Running `rustc --crate-name vte --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/vte-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=66f6ba7b673381f9 -C extra-filename=-66f6ba7b673381f9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern utf8parse=/work/oxidecomputer/crucible/target/debug/deps/libutf8parse-7be01c1e08c14495.rmeta --extern vte_generate_state_changes=/work/oxidecomputer/crucible/target/debug/deps/libvte_generate_state_changes-52ea4d6c5be5fc21.so --cap-lints allow`
     Running `/work/oxidecomputer/crucible/target/debug/build/rayon-03d683edffe821df/build-script-build`
     Running `/work/oxidecomputer/crucible/target/debug/build/terminfo-bed6d7dcce5eac66/build-script-build`
   Compiling dirs v4.0.0
     Running `rustc --crate-name dirs /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dirs-4.0.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=b6e400c266a3b937 -C extra-filename=-b6e400c266a3b937 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dirs_sys=/work/oxidecomputer/crucible/target/debug/deps/libdirs_sys-15239ab4fd89ac89.rmeta --cap-lints allow`
     Running `rustc --crate-name rayon_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rayon-core-1.9.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=000ce3790ebea6a1 -C extra-filename=-000ce3790ebea6a1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crossbeam_channel=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_channel-19ff6fa486e51c63.rmeta --extern crossbeam_deque=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_deque-46df68c0a92e69f5.rmeta --extern crossbeam_utils=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_utils-a1fb255bfa31483a.rmeta --extern num_cpus=/work/oxidecomputer/crucible/target/debug/deps/libnum_cpus-67a451bebfcc5086.rmeta --cap-lints allow`
     Running `rustc --crate-name rustix --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustix-0.33.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="io-lifetimes"' --cfg 'feature="std"' -C metadata=f2ea1fb682f42c58 -C extra-filename=-f2ea1fb682f42c58 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern errno=/work/oxidecomputer/crucible/target/debug/deps/liberrno-af3769d2acbbbf20.rmeta --extern io_lifetimes=/work/oxidecomputer/crucible/target/debug/deps/libio_lifetimes-59f01ff60b1f438f.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow --cfg libc`
   Compiling structmeta v0.2.0
     Running `rustc --crate-name structmeta --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/structmeta-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=3c927247a3e97db4 -C extra-filename=-3c927247a3e97db4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rmeta --extern structmeta_derive=/work/oxidecomputer/crucible/target/debug/deps/libstructmeta_derive-6868cfc37bbab5f5.so --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rmeta --cap-lints allow`
     Running `rustc --crate-name crc32fast /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crc32fast-1.3.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=bbced466bd6966b3 -C extra-filename=-bbced466bd6966b3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --cap-lints allow --cfg crc32fast_stdarchx86`
   Compiling phf v0.11.1
     Running `rustc --crate-name phf --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/phf-0.11.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=26f57b9bc310a574 -C extra-filename=-26f57b9bc310a574 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern phf_shared=/work/oxidecomputer/crucible/target/debug/deps/libphf_shared-f4c611c5b95ef91c.rmeta --cap-lints allow`
   Compiling tokio-stream v0.1.8
     Running `rustc --crate-name tokio_stream --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-stream-0.1.8/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="time"' -C metadata=4c3a15ae8dcea990 -C extra-filename=-4c3a15ae8dcea990 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow`
   Compiling rand_xorshift v0.3.0
     Running `rustc --crate-name rand_xorshift --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rand_xorshift-0.3.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=a099defc127d0ae4 -C extra-filename=-a099defc127d0ae4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rand_core=/work/oxidecomputer/crucible/target/debug/deps/librand_core-e2870cc0e01c33c9.rmeta --cap-lints allow`
   Compiling csv-core v0.1.10
     Running `rustc --crate-name csv_core --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/csv-core-0.1.10/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=6a47c452f1df15e2 -C extra-filename=-6a47c452f1df15e2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern memchr=/work/oxidecomputer/crucible/target/debug/deps/libmemchr-9611c546f9b73cca.rmeta --cap-lints allow`
   Compiling xattr v0.2.2
     Running `rustc --crate-name xattr /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/xattr-0.2.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="unsupported"' -C metadata=70407947526a7a4b -C extra-filename=-70407947526a7a4b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling filetime v0.2.17
     Running `rustc --crate-name filetime --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/filetime-0.2.17/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=337368c6d4c995d8 -C extra-filename=-337368c6d4c995d8 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling same-file v1.0.6
     Running `rustc --crate-name same_file --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/same-file-1.0.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=b7a6da1044557b8b -C extra-filename=-b7a6da1044557b8b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling unarray v0.1.4
     Running `rustc --crate-name unarray --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/unarray-0.1.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=bb988de2629d2530 -C extra-filename=-bb988de2629d2530 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling walkdir v2.3.2
     Running `rustc --crate-name walkdir --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/walkdir-2.3.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5232f739d2ba1b5e -C extra-filename=-5232f739d2ba1b5e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern same_file=/work/oxidecomputer/crucible/target/debug/deps/libsame_file-b7a6da1044557b8b.rmeta --cap-lints allow`
   Compiling tokio-test v0.4.2
     Running `rustc --crate-name tokio_test --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-test-0.4.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=12a28be646ff63e6 -C extra-filename=-12a28be646ff63e6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_stream=/work/oxidecomputer/crucible/target/debug/deps/libasync_stream-0486f21173e73f9c.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_stream=/work/oxidecomputer/crucible/target/debug/deps/libtokio_stream-4c3a15ae8dcea990.rmeta --cap-lints allow`
   Compiling proptest v1.2.0
     Running `rustc --crate-name proptest --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/proptest-1.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bit-set"' --cfg 'feature="break-dead-code"' --cfg 'feature="default"' --cfg 'feature="fork"' --cfg 'feature="lazy_static"' --cfg 'feature="regex-syntax"' --cfg 'feature="rusty-fork"' --cfg 'feature="std"' --cfg 'feature="tempfile"' --cfg 'feature="timeout"' -C metadata=327f7f2cf6858f27 -C extra-filename=-327f7f2cf6858f27 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bit_set=/work/oxidecomputer/crucible/target/debug/deps/libbit_set-ec12e43f0b47413c.rmeta --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rmeta --extern rand_xorshift=/work/oxidecomputer/crucible/target/debug/deps/librand_xorshift-a099defc127d0ae4.rmeta --extern regex_syntax=/work/oxidecomputer/crucible/target/debug/deps/libregex_syntax-b9593ef3338880de.rmeta --extern rusty_fork=/work/oxidecomputer/crucible/target/debug/deps/librusty_fork-20bbf553eecf6baa.rmeta --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rmeta --extern unarray=/work/oxidecomputer/crucible/target/debug/deps/libunarray-bb988de2629d2530.rmeta --cap-lints allow`
   Compiling tar v0.4.38
     Running `rustc --crate-name tar --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tar-0.4.38/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="xattr"' -C metadata=b33bc6012d78be3d -C extra-filename=-b33bc6012d78be3d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern filetime=/work/oxidecomputer/crucible/target/debug/deps/libfiletime-337368c6d4c995d8.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern xattr=/work/oxidecomputer/crucible/target/debug/deps/libxattr-70407947526a7a4b.rmeta --cap-lints allow`
   Compiling fd-lock v3.0.4
     Running `rustc --crate-name fd_lock --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fd-lock-3.0.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=dd6f5c85295045f7 -C extra-filename=-dd6f5c85295045f7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern cfg_if=/work/oxidecomputer/crucible/target/debug/deps/libcfg_if-a93276574f0edf39.rmeta --extern rustix=/work/oxidecomputer/crucible/target/debug/deps/librustix-f2ea1fb682f42c58.rmeta --cap-lints allow`
     Running `rustc --crate-name terminfo --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/terminfo-0.8.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=cd93b6cd14f79089 -C extra-filename=-cd93b6cd14f79089 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern dirs=/work/oxidecomputer/crucible/target/debug/deps/libdirs-b6e400c266a3b937.rmeta --extern fnv=/work/oxidecomputer/crucible/target/debug/deps/libfnv-5641130f60a8056b.rmeta --extern nom=/work/oxidecomputer/crucible/target/debug/deps/libnom-7d7f045245d4ae57.rmeta --extern phf=/work/oxidecomputer/crucible/target/debug/deps/libphf-26f57b9bc310a574.rmeta --cap-lints allow`
   Compiling csv v1.2.2
     Running `rustc --crate-name csv --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/csv-1.2.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=187f0e890389cec3 -C extra-filename=-187f0e890389cec3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern csv_core=/work/oxidecomputer/crucible/target/debug/deps/libcsv_core-6a47c452f1df15e2.rmeta --extern itoa=/work/oxidecomputer/crucible/target/debug/deps/libitoa-f0781104e344570e.rmeta --extern ryu=/work/oxidecomputer/crucible/target/debug/deps/libryu-062e5ac4087417b3.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --cap-lints allow`
     Running `rustc --crate-name rayon --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rayon-1.5.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=f2d40ba22c8b185e -C extra-filename=-f2d40ba22c8b185e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crossbeam_deque=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_deque-46df68c0a92e69f5.rmeta --extern either=/work/oxidecomputer/crucible/target/debug/deps/libeither-e075e5427abad613.rmeta --extern rayon_core=/work/oxidecomputer/crucible/target/debug/deps/librayon_core-000ce3790ebea6a1.rmeta --cap-lints allow --cfg has_step_by_rev --cfg has_min_const_generics --cfg has_control_flow`
   Compiling flate2 v1.0.25
     Running `rustc --crate-name flate2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/flate2-1.0.25/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="miniz_oxide"' --cfg 'feature="rust_backend"' -C metadata=e91a1b496d4e6ad4 -C extra-filename=-e91a1b496d4e6ad4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crc32fast=/work/oxidecomputer/crucible/target/debug/deps/libcrc32fast-bbced466bd6966b3.rmeta --extern miniz_oxide=/work/oxidecomputer/crucible/target/debug/deps/libminiz_oxide-f7bade0e2b1a9da4.rmeta --cap-lints allow`
   Compiling test-strategy v0.3.1
     Running `rustc --crate-name test_strategy --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/test-strategy-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type proc-macro --emit=dep-info,link -C prefer-dynamic -C embed-bitcode=no -C metadata=5eb6b90d55d9f739 -C extra-filename=-5eb6b90d55d9f739 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rlib --extern quote=/work/oxidecomputer/crucible/target/debug/deps/libquote-2ffbaa80dd156621.rlib --extern structmeta=/work/oxidecomputer/crucible/target/debug/deps/libstructmeta-3c927247a3e97db4.rlib --extern syn=/work/oxidecomputer/crucible/target/debug/deps/libsyn-baedf68a9175a6da.rlib --extern proc_macro --cap-lints allow`
   Compiling strip-ansi-escapes v0.2.0
     Running `rustc --crate-name strip_ansi_escapes /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/strip-ansi-escapes-0.2.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=8090d3de2e6bf9be -C extra-filename=-8090d3de2e6bf9be --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern vte=/work/oxidecomputer/crucible/target/debug/deps/libvte-66f6ba7b673381f9.rmeta --cap-lints allow`
   Compiling httptest v0.15.4
     Running `rustc --crate-name httptest --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/httptest-0.15.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=174da737d96e2af6 -C extra-filename=-174da737d96e2af6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bstr=/work/oxidecomputer/crucible/target/debug/deps/libbstr-f0785d87dddebfcd.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern crossbeam_channel=/work/oxidecomputer/crucible/target/debug/deps/libcrossbeam_channel-19ff6fa486e51c63.rmeta --extern form_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libform_urlencoded-ef731295a29c9150.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern regex=/work/oxidecomputer/crucible/target/debug/deps/libregex-f9e3a4eb3da387ce.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern serde_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libserde_urlencoded-779d3ac41d95557a.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow`
     Running `rustc --crate-name portable_atomic --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/portable-atomic-1.4.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="fallback"' -C metadata=f0a1a94e9d6381ba -C extra-filename=-f0a1a94e9d6381ba --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow --cfg portable_atomic_llvm_16`
   Compiling crossterm v0.26.1
     Running `rustc --crate-name crossterm --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossterm-0.26.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bracketed-paste"' --cfg 'feature="default"' --cfg 'feature="serde"' -C metadata=ddbacbe0f657f0ff -C extra-filename=-ddbacbe0f657f0ff --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern mio=/work/oxidecomputer/crucible/target/debug/deps/libmio-27a8136cf12de2bb.rmeta --extern parking_lot=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot-970d5c0acece447c.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rmeta --extern signal_hook_mio=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_mio-b78bc232ff08be19.rmeta --cap-lints allow`
   Compiling signal-hook-tokio v0.3.1
     Running `rustc --crate-name signal_hook_tokio --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/signal-hook-tokio-0.3.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="futures-core-0_3"' --cfg 'feature="futures-v0_3"' -C metadata=6a6b104c61918fa0 -C extra-filename=-6a6b104c61918fa0 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern futures_core_0_3=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow`
   Compiling itertools v0.10.5
     Running `rustc --crate-name itertools --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/itertools-0.10.5/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="use_alloc"' --cfg 'feature="use_std"' -C metadata=09aeacd112427d42 -C extra-filename=-09aeacd112427d42 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern either=/work/oxidecomputer/crucible/target/debug/deps/libeither-e075e5427abad613.rmeta --cap-lints allow`
   Compiling which v4.4.0
     Running `rustc --crate-name which --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/which-4.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=23fb4550fe083323 -C extra-filename=-23fb4550fe083323 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern either=/work/oxidecomputer/crucible/target/debug/deps/libeither-e075e5427abad613.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --cap-lints allow`
   Compiling number_prefix v0.4.0
     Running `rustc --crate-name number_prefix /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/number_prefix-0.4.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=be3728a6ec19cda2 -C extra-filename=-be3728a6ec19cda2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling nu-ansi-term v0.49.0
     Running `rustc --crate-name nu_ansi_term --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/nu-ansi-term-0.49.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=cdbbf2d007fd1e63 -C extra-filename=-cdbbf2d007fd1e63 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling utf8-width v0.1.6
     Running `rustc --crate-name utf8_width --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/utf8-width-0.1.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=7e583d5482ac364b -C extra-filename=-7e583d5482ac364b --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --cap-lints allow`
   Compiling indicatif v0.17.6
     Running `rustc --crate-name indicatif --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/indicatif-0.17.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="rayon"' --cfg 'feature="unicode-width"' -C metadata=297a26a70875006e -C extra-filename=-297a26a70875006e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern console=/work/oxidecomputer/crucible/target/debug/deps/libconsole-4236472a6e29ce0a.rmeta --extern number_prefix=/work/oxidecomputer/crucible/target/debug/deps/libnumber_prefix-be3728a6ec19cda2.rmeta --extern portable_atomic=/work/oxidecomputer/crucible/target/debug/deps/libportable_atomic-f0a1a94e9d6381ba.rmeta --extern rayon=/work/oxidecomputer/crucible/target/debug/deps/librayon-f2d40ba22c8b185e.rmeta --extern unicode_width=/work/oxidecomputer/crucible/target/debug/deps/libunicode_width-87c5262a4c4bb0e9.rmeta --cap-lints allow`
   Compiling byte-unit v4.0.19
     Running `rustc --crate-name byte_unit --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/byte-unit-4.0.19/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="serde"' --cfg 'feature="std"' --cfg 'feature="u128"' -C metadata=02cb17c857e20dac -C extra-filename=-02cb17c857e20dac --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern utf8_width=/work/oxidecomputer/crucible/target/debug/deps/libutf8_width-7e583d5482ac364b.rmeta --cap-lints allow`
   Compiling crossterm v0.27.0
     Running `rustc --crate-name crossterm --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/crossterm-0.27.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="bracketed-paste"' --cfg 'feature="default"' --cfg 'feature="events"' --cfg 'feature="windows"' -C metadata=3c787fd4c4d4bc45 -C extra-filename=-3c787fd4c4d4bc45 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-109244799287a8c3.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern mio=/work/oxidecomputer/crucible/target/debug/deps/libmio-27a8136cf12de2bb.rmeta --extern parking_lot=/work/oxidecomputer/crucible/target/debug/deps/libparking_lot-970d5c0acece447c.rmeta --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rmeta --extern signal_hook_mio=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_mio-b78bc232ff08be19.rmeta --cap-lints allow`
   Compiling reedline v0.23.0
     Running `rustc --crate-name reedline --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/reedline-0.23.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=6e6244e0f6aa654d -C extra-filename=-6e6244e0f6aa654d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern crossterm=/work/oxidecomputer/crucible/target/debug/deps/libcrossterm-ddbacbe0f657f0ff.rmeta --extern fd_lock=/work/oxidecomputer/crucible/target/debug/deps/libfd_lock-dd6f5c85295045f7.rmeta --extern itertools=/work/oxidecomputer/crucible/target/debug/deps/libitertools-09aeacd112427d42.rmeta --extern nu_ansi_term=/work/oxidecomputer/crucible/target/debug/deps/libnu_ansi_term-cdbbf2d007fd1e63.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern strip_ansi_escapes=/work/oxidecomputer/crucible/target/debug/deps/libstrip_ansi_escapes-8090d3de2e6bf9be.rmeta --extern strum=/work/oxidecomputer/crucible/target/debug/deps/libstrum-59ea3c6704348e58.rmeta --extern strum_macros=/work/oxidecomputer/crucible/target/debug/deps/libstrum_macros-bc907f623478289d.so --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern unicode_segmentation=/work/oxidecomputer/crucible/target/debug/deps/libunicode_segmentation-06176721b7b95955.rmeta --extern unicode_width=/work/oxidecomputer/crucible/target/debug/deps/libunicode_width-87c5262a4c4bb0e9.rmeta --cap-lints allow`
   Compiling nbd v0.2.3
     Running `rustc --crate-name nbd /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/nbd-0.2.3/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=6a0c01a24def5e9a -C extra-filename=-6a0c01a24def5e9a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern byteorder=/work/oxidecomputer/crucible/target/debug/deps/libbyteorder-1306999913c8e1b3.rmeta --cap-lints allow`
   Compiling crucible-integration-tests v0.1.0 (/work/oxidecomputer/crucible/integration_tests)
     Running `rustc --crate-name crucible_integration_tests --edition=2021 integration_tests/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=521d4724b4b30c4a -C extra-filename=-521d4724b4b30c4a --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps`
     Running `rustc --crate-name info --edition=2021 smf/examples/info.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 -C metadata=434b2d59a81079b5 -C extra-filename=-434b2d59a81079b5 --out-dir /work/oxidecomputer/crucible/target/debug/examples -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern num_derive=/work/oxidecomputer/crucible/target/debug/deps/libnum_derive-814c8a0a0a713cba.so --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rlib --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rlib`
   Compiling clearscreen v2.0.1
     Running `rustc --crate-name clearscreen --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/clearscreen-2.0.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5e923be7ef236a41 -C extra-filename=-5e923be7ef236a41 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rmeta --extern terminfo=/work/oxidecomputer/crucible/target/debug/deps/libterminfo-cd93b6cd14f79089.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern which=/work/oxidecomputer/crucible/target/debug/deps/libwhich-23fb4550fe083323.rmeta --cap-lints allow`
     Running `rustc --crate-name crucible_smf --edition=2021 smf/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=6114df38a9482a0c -C extra-filename=-6114df38a9482a0c --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern num_derive=/work/oxidecomputer/crucible/target/debug/deps/libnum_derive-814c8a0a0a713cba.so --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rlib --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rlib`
     Running `rustc --crate-name crucible_client_types --edition=2021 crucible-client-types/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=5ba9f9d411803900 -C extra-filename=-5ba9f9d411803900 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib`
     Running `rustc --crate-name ring --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/ring-0.16.20/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="dev_urandom_fallback"' --cfg 'feature="once_cell"' -C metadata=76ccf829b8b489e1 -C extra-filename=-76ccf829b8b489e1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern spin=/work/oxidecomputer/crucible/target/debug/deps/libspin-bfb6115ad3135235.rmeta --extern untrusted=/work/oxidecomputer/crucible/target/debug/deps/libuntrusted-4b93784238d33e58.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -l static=ring-core -l static=ring-test`
     Running `/work/oxidecomputer/crucible/target/debug/build/rustls-0c105edc866f624d/build-script-build`
   Compiling sct v0.7.0
     Running `rustc --crate-name sct --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/sct-0.7.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=78bb43c10db32a31 -C extra-filename=-78bb43c10db32a31 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ring=/work/oxidecomputer/crucible/target/debug/deps/libring-76ccf829b8b489e1.rmeta --extern untrusted=/work/oxidecomputer/crucible/target/debug/deps/libuntrusted-4b93784238d33e58.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling rustls-webpki v0.101.4
     Running `rustc --crate-name webpki --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-webpki-0.101.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="alloc"' --cfg 'feature="default"' --cfg 'feature="std"' -C metadata=34d764dbf1af1e62 -C extra-filename=-34d764dbf1af1e62 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern ring=/work/oxidecomputer/crucible/target/debug/deps/libring-76ccf829b8b489e1.rmeta --extern untrusted=/work/oxidecomputer/crucible/target/debug/deps/libuntrusted-4b93784238d33e58.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
     Running `rustc --crate-name rustls --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/rustls-0.21.6/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="dangerous_configuration"' --cfg 'feature="default"' --cfg 'feature="log"' --cfg 'feature="logging"' --cfg 'feature="tls12"' -C metadata=3df6867cfa5c4a0a -C extra-filename=-3df6867cfa5c4a0a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern ring=/work/oxidecomputer/crucible/target/debug/deps/libring-76ccf829b8b489e1.rmeta --extern webpki=/work/oxidecomputer/crucible/target/debug/deps/libwebpki-34d764dbf1af1e62.rmeta --extern sct=/work/oxidecomputer/crucible/target/debug/deps/libsct-78bb43c10db32a31.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
     Running `rustc --crate-name libgit2_sys --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libgit2-sys-0.15.2+1.6.4/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=8cfb4e998561bba5 -C extra-filename=-8cfb4e998561bba5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern libz_sys=/work/oxidecomputer/crucible/target/debug/deps/liblibz_sys-a3111f279c2174e3.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -l static=git2 -L native=/usr/lib/amd64 --cfg libgit2_vendored`
   Compiling git2 v0.17.2
     Running `rustc --crate-name git2 --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/git2-0.17.2/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C metadata=c3276e37b1dd24e5 -C extra-filename=-c3276e37b1dd24e5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bitflags=/work/oxidecomputer/crucible/target/debug/deps/libbitflags-b5bc15fb96af61fc.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern libgit2_sys=/work/oxidecomputer/crucible/target/debug/deps/liblibgit2_sys-8cfb4e998561bba5.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern url=/work/oxidecomputer/crucible/target/debug/deps/liburl-ff56943ab9066fdc.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/usr/lib/amd64`
   Compiling tokio-rustls v0.24.1
     Running `rustc --crate-name tokio_rustls --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-rustls-0.24.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' --cfg 'feature="logging"' --cfg 'feature="tls12"' -C metadata=eafe4ab74a176b7d -C extra-filename=-eafe4ab74a176b7d --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern rustls=/work/oxidecomputer/crucible/target/debug/deps/librustls-3df6867cfa5c4a0a.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling hyper-rustls v0.24.0
     Running `rustc --crate-name hyper_rustls --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hyper-rustls-0.24.0/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=9a83fdcd4675665e -C extra-filename=-9a83fdcd4675665e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern rustls=/work/oxidecomputer/crucible/target/debug/deps/librustls-3df6867cfa5c4a0a.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
     Running `rustc --crate-name dropshot --edition=2018 /home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/aca6de3/dropshot/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="usdt"' --cfg 'feature="usdt-probes"' -C metadata=a49a4505c9c6b86f -C extra-filename=-a49a4505c9c6b86f --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern async_stream=/work/oxidecomputer/crucible/target/debug/deps/libasync_stream-0486f21173e73f9c.rmeta --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern camino=/work/oxidecomputer/crucible/target/debug/deps/libcamino-45f0f4a2c258f934.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern debug_ignore=/work/oxidecomputer/crucible/target/debug/deps/libdebug_ignore-2303f500fcbc7093.rmeta --extern dropshot_endpoint=/work/oxidecomputer/crucible/target/debug/deps/libdropshot_endpoint-1ff3a3dd0352c250.so --extern form_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libform_urlencoded-ef731295a29c9150.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern hostname=/work/oxidecomputer/crucible/target/debug/deps/libhostname-4a0f8b1a56e5681a.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern indexmap=/work/oxidecomputer/crucible/target/debug/deps/libindexmap-c2c8f74266eebb64.rmeta --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rmeta --extern paste=/work/oxidecomputer/crucible/target/debug/deps/libpaste-251489637fc3d2bc.so --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern proc_macro2=/work/oxidecomputer/crucible/target/debug/deps/libproc_macro2-a0e1129b171da08c.rmeta --extern rustls=/work/oxidecomputer/crucible/target/debug/deps/librustls-3df6867cfa5c4a0a.rmeta --extern rustls_pemfile=/work/oxidecomputer/crucible/target/debug/deps/librustls_pemfile-e52b2a6debfcae48.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern serde_path_to_error=/work/oxidecomputer/crucible/target/debug/deps/libserde_path_to_error-f8fffad8b554a310.rmeta --extern serde_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libserde_urlencoded-779d3ac41d95557a.rmeta --extern sha1=/work/oxidecomputer/crucible/target/debug/deps/libsha1-8fdbd7a715f3bef1.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rmeta --extern slog_bunyan=/work/oxidecomputer/crucible/target/debug/deps/libslog_bunyan-dce051a6775f1d99.rmeta --extern slog_json=/work/oxidecomputer/crucible/target/debug/deps/libslog_json-d8408f8f3a6dd5b7.rmeta --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --extern waitgroup=/work/oxidecomputer/crucible/target/debug/deps/libwaitgroup-db859ead02bd709e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling reqwest v0.11.20
     Running `rustc --crate-name reqwest --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/reqwest-0.11.20/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="__rustls"' --cfg 'feature="__tls"' --cfg 'feature="blocking"' --cfg 'feature="default"' --cfg 'feature="default-tls"' --cfg 'feature="hyper-rustls"' --cfg 'feature="hyper-tls"' --cfg 'feature="json"' --cfg 'feature="native-tls-crate"' --cfg 'feature="rustls"' --cfg 'feature="rustls-pemfile"' --cfg 'feature="rustls-tls"' --cfg 'feature="rustls-tls-webpki-roots"' --cfg 'feature="serde_json"' --cfg 'feature="stream"' --cfg 'feature="tokio-native-tls"' --cfg 'feature="tokio-rustls"' --cfg 'feature="tokio-util"' --cfg 'feature="wasm-streams"' --cfg 'feature="webpki-roots"' -C metadata=6407fc4e9374ca8e -C extra-filename=-6407fc4e9374ca8e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern encoding_rs=/work/oxidecomputer/crucible/target/debug/deps/libencoding_rs-3255048793b3f7a6.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern h2=/work/oxidecomputer/crucible/target/debug/deps/libh2-3e2d8390f23dd48a.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern http_body=/work/oxidecomputer/crucible/target/debug/deps/libhttp_body-bb1d69dd918c127f.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern hyper_rustls=/work/oxidecomputer/crucible/target/debug/deps/libhyper_rustls-9a83fdcd4675665e.rmeta --extern hyper_tls=/work/oxidecomputer/crucible/target/debug/deps/libhyper_tls-2dbf57f91f681e2c.rmeta --extern ipnet=/work/oxidecomputer/crucible/target/debug/deps/libipnet-8b250db103a32779.rmeta --extern log=/work/oxidecomputer/crucible/target/debug/deps/liblog-930e200cffaa7930.rmeta --extern mime=/work/oxidecomputer/crucible/target/debug/deps/libmime-ac14a9115eddd3c2.rmeta --extern native_tls_crate=/work/oxidecomputer/crucible/target/debug/deps/libnative_tls-320c05ab5bbd33c9.rmeta --extern once_cell=/work/oxidecomputer/crucible/target/debug/deps/libonce_cell-fdb5f9e769d1e589.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern pin_project_lite=/work/oxidecomputer/crucible/target/debug/deps/libpin_project_lite-c035e320730303c2.rmeta --extern rustls=/work/oxidecomputer/crucible/target/debug/deps/librustls-3df6867cfa5c4a0a.rmeta --extern rustls_pemfile=/work/oxidecomputer/crucible/target/debug/deps/librustls_pemfile-e52b2a6debfcae48.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern serde_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libserde_urlencoded-779d3ac41d95557a.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_native_tls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_native_tls-f56aba82a642e205.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --extern tower_service=/work/oxidecomputer/crucible/target/debug/deps/libtower_service-51da71f2ad5117ee.rmeta --extern url=/work/oxidecomputer/crucible/target/debug/deps/liburl-ff56943ab9066fdc.rmeta --extern webpki_roots=/work/oxidecomputer/crucible/target/debug/deps/libwebpki_roots-31272bd9a7615638.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling progenitor-client v0.3.0 (https://github.com/oxidecomputer/progenitor?branch=main#76716eea)
   Compiling omicron-zone-package v0.9.1
     Running `rustc --crate-name progenitor_client --edition=2021 /home/build/.cargo/git/checkouts/progenitor-639bd64206ac5e43/76716ee/progenitor-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=a49921826ff8ec2a -C extra-filename=-a49921826ff8ec2a --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern serde_urlencoded=/work/oxidecomputer/crucible/target/debug/deps/libserde_urlencoded-779d3ac41d95557a.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name omicron_zone_package --edition=2018 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/omicron-zone-package-0.9.1/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=8423a7bf8bd88040 -C extra-filename=-8423a7bf8bd88040 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern filetime=/work/oxidecomputer/crucible/target/debug/deps/libfiletime-337368c6d4c995d8.rmeta --extern flate2=/work/oxidecomputer/crucible/target/debug/deps/libflate2-e91a1b496d4e6ad4.rmeta --extern futures_util=/work/oxidecomputer/crucible/target/debug/deps/libfutures_util-b4da5d5433271d56.rmeta --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern ring=/work/oxidecomputer/crucible/target/debug/deps/libring-76ccf829b8b489e1.rmeta --extern semver=/work/oxidecomputer/crucible/target/debug/deps/libsemver-8c1c5827befd93e7.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_derive=/work/oxidecomputer/crucible/target/debug/deps/libserde_derive-3441a1f9756a6d5b.so --extern tar=/work/oxidecomputer/crucible/target/debug/deps/libtar-b33bc6012d78be3d.rmeta --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern walkdir=/work/oxidecomputer/crucible/target/debug/deps/libwalkdir-5232f739d2ba1b5e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling progenitor v0.3.0 (https://github.com/oxidecomputer/progenitor?branch=main#76716eea)
     Running `rustc --crate-name progenitor --edition=2021 /home/build/.cargo/git/checkouts/progenitor-639bd64206ac5e43/76716ee/progenitor/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0319e0dfd841f493 -C extra-filename=-0319e0dfd841f493 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern progenitor_client=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor_client-a49921826ff8ec2a.rmeta --extern progenitor_impl=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor_impl-4eeb9d4c0f08c454.rmeta --extern progenitor_macro=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor_macro-4c13a8353939c841.so --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling repair-client v0.0.1 (/work/oxidecomputer/crucible/repair-client)
     Running `rustc --crate-name repair_client --edition=2021 repair-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=1452d56087b6ccb7 -C extra-filename=-1452d56087b6ccb7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-pantry-client v0.0.1 (/work/oxidecomputer/crucible/pantry-client)
     Running `rustc --crate-name crucible_pantry_client --edition=2021 pantry-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=ccb9ddeebb23cea2 -C extra-filename=-ccb9ddeebb23cea2 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling dsc-client v0.0.1 (/work/oxidecomputer/crucible/dsc-client)
     Running `rustc --crate-name dsc_client --edition=2021 dsc-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=15b0c81fa833cf0f -C extra-filename=-15b0c81fa833cf0f --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name vergen --edition=2021 /home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/vergen-8.2.4/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no --cfg 'feature="cargo"' --cfg 'feature="default"' --cfg 'feature="git"' --cfg 'feature="git2"' --cfg 'feature="git2-rs"' --cfg 'feature="rustc"' --cfg 'feature="rustc_version"' --cfg 'feature="time"' -C metadata=e75c33287bd2547c -C extra-filename=-e75c33287bd2547c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern git2_rs=/work/oxidecomputer/crucible/target/debug/deps/libgit2-c3276e37b1dd24e5.rmeta --extern rustc_version=/work/oxidecomputer/crucible/target/debug/deps/librustc_version-201ef6100eba532b.rmeta --extern time=/work/oxidecomputer/crucible/target/debug/deps/libtime-9b604407a0d52f86.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/usr/lib/amd64 --cfg stable --cfg msrv`
   Compiling omicron-common v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name omicron_common --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/common/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=52df1fff8b36d94c -C extra-filename=-52df1fff8b36d94c --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern api_identity=/work/oxidecomputer/crucible/target/debug/deps/libapi_identity-90d45ecc06c8f773.so --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern backoff=/work/oxidecomputer/crucible/target/debug/deps/libbackoff-2bc4a2fd075cf434.rmeta --extern camino=/work/oxidecomputer/crucible/target/debug/deps/libcamino-45f0f4a2c258f934.rmeta --extern camino_tempfile=/work/oxidecomputer/crucible/target/debug/deps/libcamino_tempfile-38b6a8d85c9dc0c0.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern ipnetwork=/work/oxidecomputer/crucible/target/debug/deps/libipnetwork-0e9e550a49db2c52.rmeta --extern lazy_static=/work/oxidecomputer/crucible/target/debug/deps/liblazy_static-9c1a21fa7aad0259.rmeta --extern macaddr=/work/oxidecomputer/crucible/target/debug/deps/libmacaddr-98e89df75c36be48.rmeta --extern parse_display=/work/oxidecomputer/crucible/target/debug/deps/libparse_display-34a1a5d52375b70b.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern ring=/work/oxidecomputer/crucible/target/debug/deps/libring-76ccf829b8b489e1.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern semver=/work/oxidecomputer/crucible/target/debug/deps/libsemver-8c1c5827befd93e7.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_derive=/work/oxidecomputer/crucible/target/debug/deps/libserde_derive-3441a1f9756a6d5b.so --extern serde_human_bytes=/work/oxidecomputer/crucible/target/debug/deps/libserde_human_bytes-7f54d0fcbf9b36f3.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern serde_with=/work/oxidecomputer/crucible/target/debug/deps/libserde_with-4f9ddd30b380d6cf.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern steno=/work/oxidecomputer/crucible/target/debug/deps/libsteno-d1d3ce30296926ad.rmeta --extern strum=/work/oxidecomputer/crucible/target/debug/deps/libstrum-59ea3c6704348e58.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_postgres=/work/oxidecomputer/crucible/target/debug/deps/libtokio_postgres-5628b93feb58339b.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-control-client v0.0.1 (/work/oxidecomputer/crucible/control-client)
     Running `rustc --crate-name crucible_control_client --edition=2021 control-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=3d0142c7d3790e17 -C extra-filename=-3d0142c7d3790e17 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-agent-client v0.0.1 (/work/oxidecomputer/crucible/agent-client)
     Running `rustc --crate-name crucible_agent_client --edition=2021 agent-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=86e1c18945d61be3 -C extra-filename=-86e1c18945d61be3 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-common v0.0.1 (/work/oxidecomputer/crucible/common)
     Running `rustc --crate-name build_script_build --edition=2021 common/build.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type bin --emit=dep-info,link -C embed-bitcode=no -C metadata=e7c30bee09b7ef3b -C extra-filename=-e7c30bee09b7ef3b --out-dir /work/oxidecomputer/crucible/target/debug/build/crucible-common-e7c30bee09b7ef3b -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern vergen=/work/oxidecomputer/crucible/target/debug/deps/libvergen-e75c33287bd2547c.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/usr/lib/amd64`
     Running `/work/oxidecomputer/crucible/target/debug/build/crucible-common-e7c30bee09b7ef3b/build-script-build`
     Running `rustc --crate-name crucible_common --edition=2021 common/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=c45901e9152d33f4 -C extra-filename=-c45901e9152d33f4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern atty=/work/oxidecomputer/crucible/target/debug/deps/libatty-bfb6a2cdc762f7c4.rmeta --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rmeta --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rmeta --extern rustls_pemfile=/work/oxidecomputer/crucible/target/debug/deps/librustls_pemfile-e52b2a6debfcae48.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rmeta --extern slog_bunyan=/work/oxidecomputer/crucible/target/debug/deps/libslog_bunyan-dce051a6775f1d99.rmeta --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rmeta --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rmeta --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern twox_hash=/work/oxidecomputer/crucible/target/debug/deps/libtwox_hash-9f5dd4f7319ca539.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
     Running `rustc --crate-name crucible_common --edition=2021 common/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=9a32809bdbdf85c4 -C extra-filename=-9a32809bdbdf85c4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern atty=/work/oxidecomputer/crucible/target/debug/deps/libatty-bfb6a2cdc762f7c4.rlib --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rlib --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rlib --extern rustls_pemfile=/work/oxidecomputer/crucible/target/debug/deps/librustls_pemfile-e52b2a6debfcae48.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_bunyan=/work/oxidecomputer/crucible/target/debug/deps/libslog_bunyan-dce051a6775f1d99.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern twox_hash=/work/oxidecomputer/crucible/target/debug/deps/libtwox_hash-9f5dd4f7319ca539.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling crucible-protocol v0.0.0 (/work/oxidecomputer/crucible/protocol)
     Running `rustc --crate-name crucible_protocol --edition=2021 protocol/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=0682e169c907a102 -C extra-filename=-0682e169c907a102 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rmeta --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern num_enum=/work/oxidecomputer/crucible/target/debug/deps/libnum_enum-9cd7a6d9dcf1dd5a.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
     Running `rustc --crate-name crucible_protocol --edition=2021 protocol/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=c776b78ce4b42bf6 -C extra-filename=-c776b78ce4b42bf6 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern num_enum=/work/oxidecomputer/crucible/target/debug/deps/libnum_enum-9cd7a6d9dcf1dd5a.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling dsc v0.1.0 (/work/oxidecomputer/crucible/dsc)
     Running `rustc --crate-name dsc_client --edition=2021 dsc-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=f8a5b497695371e1 -C extra-filename=-f8a5b497695371e1 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name repair_client --edition=2021 repair-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=5353c8de97b4615f -C extra-filename=-5353c8de97b4615f --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name dsc --edition=2021 dsc/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=2b80980cbc3bac2c -C extra-filename=-2b80980cbc3bac2c --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern byte_unit=/work/oxidecomputer/crucible/target/debug/deps/libbyte_unit-02cb17c857e20dac.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern csv=/work/oxidecomputer/crucible/target/debug/deps/libcsv-187f0e890389cec3.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern dsc_client=/work/oxidecomputer/crucible/target/debug/deps/libdsc_client-15b0c81fa833cf0f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_agent_client --edition=2021 agent-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=6f3900e8033b57ec -C extra-filename=-6f3900e8033b57ec --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-package v0.1.0 (/work/oxidecomputer/crucible/package)
     Running `rustc --crate-name crucible_package --edition=2021 package/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=e8ff0170d25e0da5 -C extra-filename=-e8ff0170d25e0da5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern omicron_zone_package=/work/oxidecomputer/crucible/target/debug/deps/libomicron_zone_package-8423a7bf8bd88040.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling oximeter v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name oximeter --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/oximeter/oximeter/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=90ae047d6b643e4e -C extra-filename=-90ae047d6b643e4e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern oximeter_macro_impl=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_macro_impl-e4cc949eda20c416.so --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling nexus-client v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name nexus_client --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/nexus-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fd6034b144d15fe8 -C extra-filename=-fd6034b144d15fe8 --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern ipnetwork=/work/oxidecomputer/crucible/target/debug/deps/libipnetwork-0e9e550a49db2c52.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern omicron_passwords=/work/oxidecomputer/crucible/target/debug/deps/libomicron_passwords-ac6e3a602e6ad041.rmeta --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rmeta --extern regress=/work/oxidecomputer/crucible/target/debug/deps/libregress-10da65958da1c830.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_control_client --edition=2021 control-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=d0a58354872d46d9 -C extra-filename=-d0a58354872d46d9 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_pantry_client --edition=2021 pantry-client/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=8a27f01eb086219e -C extra-filename=-8a27f01eb086219e --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_protocol --edition=2021 protocol/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=d81e6562be2ffe77 -C extra-filename=-d81e6562be2ffe77 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern num_enum=/work/oxidecomputer/crucible/target/debug/deps/libnum_enum-9cd7a6d9dcf1dd5a.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out`
   Compiling crucible-agent v0.0.1 (/work/oxidecomputer/crucible/agent)
     Running `rustc --crate-name crucible_agent --edition=2021 agent/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=acbf118b39db417b -C extra-filename=-acbf118b39db417b --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern subprocess=/work/oxidecomputer/crucible/target/debug/deps/libsubprocess-0acfc5c9b903588a.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling oximeter-producer v0.1.0 (https://github.com/oxidecomputer/omicron?branch=main#4c05962d)
     Running `rustc --crate-name oximeter_producer --edition=2021 /home/build/.cargo/git/checkouts/omicron-d039c41f152bda83/4c05962/oximeter/producer/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=5dc4f732e258486e -C extra-filename=-5dc4f732e258486e --out-dir /work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rmeta --extern nexus_client=/work/oxidecomputer/crucible/target/debug/deps/libnexus_client-fd6034b144d15fe8.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rmeta --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta --cap-lints allow -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible --edition=2021 upstairs/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=41ca439abdc23695 -C extra-filename=-41ca439abdc23695 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern aes_gcm_siv=/work/oxidecomputer/crucible/target/debug/deps/libaes_gcm_siv-21495b616a07c9a4.rmeta --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern async_recursion=/work/oxidecomputer/crucible/target/debug/deps/libasync_recursion-ce9499495a1cb858.so --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rmeta --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rmeta --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rmeta --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern itertools=/work/oxidecomputer/crucible/target/debug/deps/libitertools-b06e69badd72e55c.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rmeta --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rmeta --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rmeta --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible --edition=2021 upstairs/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=53d074fabbf363e8 -C extra-filename=-53d074fabbf363e8 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern aes_gcm_siv=/work/oxidecomputer/crucible/target/debug/deps/libaes_gcm_siv-21495b616a07c9a4.rlib --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern async_recursion=/work/oxidecomputer/crucible/target/debug/deps/libasync_recursion-ce9499495a1cb858.so --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern itertools=/work/oxidecomputer/crucible/target/debug/deps/libitertools-b06e69badd72e55c.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern proptest=/work/oxidecomputer/crucible/target/debug/deps/libproptest-327f7f2cf6858f27.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern test_strategy=/work/oxidecomputer/crucible/target/debug/deps/libtest_strategy-5eb6b90d55d9f739.so --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern tokio_test=/work/oxidecomputer/crucible/target/debug/deps/libtokio_test-12a28be646ff63e6.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rlib --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-pantry v0.0.1 (/work/oxidecomputer/crucible/pantry)
     Running `rustc --crate-name crucible_pantry --edition=2021 pantry/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 -C metadata=fe633af5059fe3a7 -C extra-filename=-fe633af5059fe3a7 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rmeta --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rmeta --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rmeta --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rmeta --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_downstairs --edition=2021 downstairs/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debuginfo=2 --cfg 'feature="default"' -C metadata=6276be71be5284a4 -C extra-filename=-6276be71be5284a4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rmeta --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rmeta --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rmeta --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rmeta --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rmeta --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rmeta --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rmeta --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rmeta --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rmeta --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rmeta --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rmeta --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rmeta --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rmeta --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rmeta --extern hyper_staticfile=/work/oxidecomputer/crucible/target/debug/deps/libhyper_staticfile-559b4389ef952563.rmeta --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rmeta --extern mime_guess=/work/oxidecomputer/crucible/target/debug/deps/libmime_guess-66974d6c31968dc2.rmeta --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rmeta --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rmeta --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-5524fd7817ad57fb.rmeta --extern opentelemetry_jaeger=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_jaeger-9ebf47742e5e063f.rmeta --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rmeta --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rmeta --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rmeta --extern repair_client=/work/oxidecomputer/crucible/target/debug/deps/librepair_client-1452d56087b6ccb7.rmeta --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rmeta --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rmeta --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rmeta --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rmeta --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rmeta --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rmeta --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rmeta --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rmeta --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rmeta --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rmeta --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rmeta --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rmeta --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rmeta --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rmeta --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rmeta --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rmeta --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rmeta --extern tracing_opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libtracing_opentelemetry-8f8cfc1900c3a663.rmeta --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rmeta --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rmeta --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rmeta -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-hammer v0.1.0 (/work/oxidecomputer/crucible/hammer)
   Compiling cmon v0.1.0 (/work/oxidecomputer/crucible/cmon)
   Compiling measure-iops v0.0.1 (/work/oxidecomputer/crucible/measure_iops)
     Running `rustc --crate-name crucible_pantry --edition=2021 pantry/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=b51bc30f7a0cbfa5 -C extra-filename=-b51bc30f7a0cbfa5 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_pantry=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry-fe633af5059fe3a7.rlib --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern subprocess=/work/oxidecomputer/crucible/target/debug/deps/libsubprocess-0acfc5c9b903588a.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_hammer --edition=2021 hammer/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=9622fb9be260fb45 -C extra-filename=-9622fb9be260fb45 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-5524fd7817ad57fb.rlib --extern opentelemetry_jaeger=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_jaeger-9ebf47742e5e063f.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tracing_opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libtracing_opentelemetry-8f8cfc1900c3a663.rlib --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name measure_iops --edition=2021 measure_iops/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=cbdca99bf515defe -C extra-filename=-cbdca99bf515defe --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_downstairs --edition=2021 downstairs/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test --cfg 'feature="default"' -C metadata=dce67baac661a5f4 -C extra-filename=-dce67baac661a5f4 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern hyper_staticfile=/work/oxidecomputer/crucible/target/debug/deps/libhyper_staticfile-559b4389ef952563.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern mime_guess=/work/oxidecomputer/crucible/target/debug/deps/libmime_guess-66974d6c31968dc2.rlib --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-5524fd7817ad57fb.rlib --extern opentelemetry_jaeger=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_jaeger-9ebf47742e5e063f.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern repair_client=/work/oxidecomputer/crucible/target/debug/deps/librepair_client-1452d56087b6ccb7.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rlib --extern tracing_opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libtracing_opentelemetry-8f8cfc1900c3a663.rlib --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rlib --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name cmon --edition=2021 cmon/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=2a9909624d24c98d -C extra-filename=-2a9909624d24c98d --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern clearscreen=/work/oxidecomputer/crucible/target/debug/deps/libclearscreen-5e923be7ef236a41.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_control_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_control_client-3d0142c7d3790e17.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_pantry --edition=2021 pantry/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=8e1bf492bfe90e8c -C extra-filename=-8e1bf492bfe90e8c --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern subprocess=/work/oxidecomputer/crucible/target/debug/deps/libsubprocess-0acfc5c9b903588a.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crudd v0.1.0 (/work/oxidecomputer/crucible/crudd)
     Running `rustc --crate-name crudd --edition=2021 crudd/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=3e9b00990c25260e -C extra-filename=-3e9b00990c25260e --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rlib --extern signal_hook_tokio=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_tokio-6a6b104c61918fa0.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crucible-nbd-server v0.1.0 (/work/oxidecomputer/crucible/nbd_server)
     Running `rustc --crate-name crucible_nbd_server --edition=2021 nbd_server/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=8c1612631a1669fd -C extra-filename=-8c1612631a1669fd --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern nbd=/work/oxidecomputer/crucible/target/debug/deps/libnbd-6a0c01a24def5e9a.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
   Compiling crutest v0.1.0 (/work/oxidecomputer/crucible/crutest)
     Running `rustc --crate-name crutest --edition=2021 crutest/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=af78e92d646e2d06 -C extra-filename=-af78e92d646e2d06 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crossterm=/work/oxidecomputer/crucible/target/debug/deps/libcrossterm-3c787fd4c4d4bc45.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern csv=/work/oxidecomputer/crucible/target/debug/deps/libcsv-187f0e890389cec3.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern dsc_client=/work/oxidecomputer/crucible/target/debug/deps/libdsc_client-15b0c81fa833cf0f.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern indicatif=/work/oxidecomputer/crucible/target/debug/deps/libindicatif-297a26a70875006e.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern reedline=/work/oxidecomputer/crucible/target/debug/deps/libreedline-6e6244e0f6aa654d.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern signal_hook=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook-ae206b38743b6815.rlib --extern signal_hook_tokio=/work/oxidecomputer/crucible/target/debug/deps/libsignal_hook_tokio-6a6b104c61918fa0.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_integration_tests --edition=2021 integration_tests/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test -C metadata=8902d603847d3610 -C extra-filename=-8902d603847d3610 --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rlib --extern crucible_downstairs=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_downstairs-6276be71be5284a4.rlib --extern crucible_pantry=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry-fe633af5059fe3a7.rlib --extern crucible_pantry_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry_client-ccb9ddeebb23cea2.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern httptest=/work/oxidecomputer/crucible/target/debug/deps/libhttptest-174da737d96e2af6.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
     Running `rustc --crate-name crucible_downstairs --edition=2021 downstairs/src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --emit=dep-info,link -C embed-bitcode=no -C debuginfo=2 --test --cfg 'feature="default"' -C metadata=3ed9735920c1592d -C extra-filename=-3ed9735920c1592d --out-dir /work/oxidecomputer/crucible/target/debug/deps -C incremental=/work/oxidecomputer/crucible/target/debug/incremental -L dependency=/work/oxidecomputer/crucible/target/debug/deps --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_downstairs=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_downstairs-6276be71be5284a4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern hyper_staticfile=/work/oxidecomputer/crucible/target/debug/deps/libhyper_staticfile-559b4389ef952563.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern mime_guess=/work/oxidecomputer/crucible/target/debug/deps/libmime_guess-66974d6c31968dc2.rlib --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-5524fd7817ad57fb.rlib --extern opentelemetry_jaeger=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_jaeger-9ebf47742e5e063f.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern repair_client=/work/oxidecomputer/crucible/target/debug/deps/librepair_client-1452d56087b6ccb7.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rlib --extern tracing_opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libtracing_opentelemetry-8f8cfc1900c3a663.rlib --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rlib --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -L native=/usr/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out -L native=/usr/ssl-3/lib/amd64`
    Finished test [unoptimized + debuginfo] target(s) in 5m 36s
     Running `/work/oxidecomputer/crucible/target/debug/deps/cmon-2a9909624d24c98d --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible-53d074fabbf363e8 --nocapture`

running 351 tests
test block_req::test::test_blockreq_and_blockreqwaiter_err ... ok
test block_req::test::test_blockreqwaiter_send ... ok
test block_req::test::test_blockreq_and_blockreqwaiter ... ok
{{{"msg""msg"::""Upstairs startsUpstairs starts""{,",v""v""msg":"Upstairs starts":,:"v":000,,""name"name:"":,"crucible""name"crucible":"crucible",,""levellevel""::,"level":303030"msg":"Upstairs starts","v":0,"name":"crucible","level":30,",,"time""time":":time"":"2023-09-22T23:14:58.571033243Z"2023-09-22T23:14:58.571099761Z"2023-09-22T23:14:58.570989567Z",",",hostname":""hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,"pid"pidpid""::"47594759:4759,"upstairs":,"1upstairs":1}
}
{{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: "\"msg":"x86_64-unknown-illumos\",\nCrucible Version: BuildInfo {    debug: true,\n\n    version: \"    opt_level: 0,0.0.1\n\"},"\n    git_sha: \",,""v"ed48f294784d46ea7d4bb99336918b74358eca46\":upstairs0",:,"\nname1":"    git_commit_timestamp: }crucible"
,\""level":2023-09-22T22:51:18.000000000Z\"30,\n{    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n"msg":    rustc_channel: \""stable\",\nCrucible Version: BuildInfo {\n    rustc_host_triple: \"    version: \"x86_64-unknown-illumos0.0.1\"\",\n,\n    git_sha: \"    rustc_commit_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n,"90c541806f23a127002de5b4038be731ba1458ca    git_commit_timestamp: time\""\":2023-09-22T22:51:18.000000000Z\"",\n,\n    git_branch: 2023-09-22T23:14:58.571766951Z\""main\"    cargo_triple: \",,\n"x86_64-unknown-illumos\"hostname    rustc_semver: "\":"1.70.0\",,ip-10-150-1-55.us-west-2.compute.internal"\n,"\npid    rustc_channel: "\":    debug: true,\n4759stable\",    opt_level: 0,\n,"\n    rustc_host_triple: upstairs}""\":1}x86_64-unknown-illumos
\",,\n"v":    rustc_commit_sha: \"0{,"name"90c541806f23a127002de5b4038be731ba1458ca"\":"msg,"\n:"crucible"    cargo_triple: \",Upstairs <-> Downstairs Message Version: 4x86_64-unknown-illumos\"",\n,""    debug: true,v\n":level0    opt_level: 0,,\n"}name"""::,""30cruciblev"":,"0level",:"30name":"crucible","level":30,"time":"2023-09-22T23:14:58.571840911Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time",:""pid,":2023-09-22T23:14:58.571844237Z4759",""time,upstairs"":hostname1"}"
:":"{ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.571839079Z","",msgpid""::""4759hostname":","Crucible stats registered with UUID: 28a12560-c07c-41d3-8d35-7e06b0c337aaupstairs"":1,ip-10-150-1-55.us-west-2.compute.internal"}"
v":,0"{,pid"":"name":msg""crucible":,""4759level"Upstairs <-> Downstairs Message Version: 4:"30,,""vupstairs""::10,}"
,name"":time"":"crucible","level2023-09-22T23:14:58.571895162Z"":30,"hostname"{:"ip-10-150-1-55.us-west-2.compute.internal",""pidmsg":"":,4759"time",:""Upstairs <-> Downstairs Message Version: 4"upstairs2023-09-22T23:14:58.571911239Z"",",:"1v"hostname}"
::"0{,"name"ip-10-150-1-55.us-west-2.compute.internal""msg,"":pid""::"4759crucible",,"Crucible 28a12560-c07c-41d3-8d35-7e06b0c337aa has session id: 85a5640b-a5e5-4a5f-a0f6-5c5b8b866c68"upstairs",:"1v"}:
0","{name":"level"cruciblemsg"":",""level"::Crucible stats registered with UUID: 04482c5c-f564-49f3-897c-51c4de58239a30"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.571966155Z","hostname":,"","time"ip-10-150-1-55.us-west-2.compute.internal:"","time"pid2023-09-22T23:14:58.571971701Z"":,4759":"hostname,"":"upstairs"2023-09-22T23:14:58.571967359Z":ip-10-150-1-55.us-west-2.compute.internal"1,,""}hostname
pid"":":4759,"ip-10-150-1-55.us-west-2.compute.internal"upstairs,""pid"::47591}
,"upstairs":{1}
"msg":"{Crucible 04482c5c-f564-49f3-897c-51c4de58239a has session id: a9998b98-45c5-45f8-8be8-f4cf49aa5402"",msg"":v"":0,"name":"crucible",Crucible stats registered with UUID: 85682a66-27fd-403c-bf22-d04b4ff66805""level":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572032802Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time,""pid":":47592023-09-22T23:14:58.572038155Z",",upstairs":"1hostname"}:"
ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Crucible 85682a66-27fd-403c-bf22-d04b4ff66805 has session id: 498b136b-9c8b-4715-a815-2b4d0646e0b2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572092695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,"time":"2023-09-22T23:14:58.570990413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572185432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572230815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Crucible stats registered with UUID: 5c1ab357-2b46-4e37-877a-5089e53cee0e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572272053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Crucible 5c1ab357-2b46-4e37-877a-5089e53cee0e has session id: 7f7beff6-5728-4471-a291-79266d94fadb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572315205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] connecting to 127.0.0.1:58766","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572522249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] connecting to 127.0.0.1:39040","v":0,"name":"crucible","level":30,"{time":"2023-09-22T23:14:58.57260439Z",""hostname":"msg":"ip-10-150-1-55.us-west-2.compute.internal","[0] connecting to 127.0.0.1:36984"pid":4759,"v":,"0looper":","0name":""crucible",","upstairs"level"::130}
{"msg":"[1] connecting to 127.0.0.1:62630","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572646323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
,"time":"2023-09-22T23:14:58.572670529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"1","upstairs":1}
{"msg":"[2] connecting to 127.0.0.1:41929","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572846053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"2","upstairs":1}
{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572915735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.572970306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Flush timeout: 86400","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573012072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] connecting to 127.0.0.1:36921","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573096855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[1] connecting to 127.0.0.1:37306","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573143501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"1","upstairs":1}
{"msg":"[1] connecting to 127.0.0.1:47390","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573226179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"1","upstairs":1}
{"msg":"[2] connecting to 127.0.0.1:35000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573271604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"2","upstairs":1}
{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573313456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1{,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30"msg":"[1] connecting to 127.0.0.1:46704",","time"v":":0,2023-09-22T23:14:58.573413634Z""name":","crucible"hostname":","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{,"time":""msg":"2023-09-22T23:14:58.573447929Z"Flush timeout: 86400",","hostnamev":":"0,"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible",","pid"level"::475930,"looper":"1","upstairs":1}
{,"time"":msg":""[2] connecting to 127.0.0.1:611412023-09-22T23:14:58.573484133Z"",","v"hostname"::"0,"nameip-10-150-1-55.us-west-2.compute.internal"":,""pid":crucible4759",,""upstairs":level1":}30
,"time":"2023-09-22T23:14:58.573526064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"2","upstairs":1}
{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573572808Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573616769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Flush timeout: 86400","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573656646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] 28a12560-c07c-41d3-8d35-7e06b0c337aa looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573697367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759{,"looper":"1","upstairs":1"}
msg":"{[0] 5c1ab357-2b46-4e37-877a-5089e53cee0e looper connected"","msg"v"::"0,"name":"[1] Proc runs for 127.0.0.1:46704 in state New"crucible",",v"":level0",:"30name":"crucible","level":30,"time":"2023-09-22T23:14:58.573750561Z","hostname,":""time":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.573747831Z"",","pid"hostname"::"4759,"ip-10-150-1-55.us-west-2.compute.internal"upstairs,":"1pid":}
4759,"looper":"0","upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:58766 in state New","v":{0,"name":"crucible"","msg":level":"30{[2] connecting to 127.0.0.1:55683","v":0,""name":"msg"crucible":,""level":30[0] 85682a66-27fd-403c-bf22-d04b4ff66805 looper connected","v":0,,""name":time":""crucible"2023-09-22T23:14:58.573822503Z,""level",:"30,hostname"":time"":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.573832561Z","pid,"":hostname4759":","upstairs":1ip-10-150-1-55.us-west-2.compute.internal"},
","pid"time"::"47592023-09-22T23:14:58.573848094Z,""looper",":"hostname2"":,""upstairs":1ip-10-150-1-55.us-west-2.compute.internal"},"
pid":4759,"{looper":"0",""upstairsmsg""::"1up_listen starts"},
"v":0,"name":"crucible"{,"level":30"msg":"[0] Proc runs for 127.0.0.1:39040 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573916202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":,"1time":","task":2023-09-22T23:14:58.573925761Z"","up_listen"hostname}"
:"ip-10-150-1-55.us-west-2.compute.internal","{pid":4759","msg"upstairs:"":1}
Wait for all three downstairs to come online","v":0,"{name":"crucible",""levelmsg"":":30[1] 85682a66-27fd-403c-bf22-d04b4ff66805 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.573977041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":,"1time":"}
2023-09-22T23:14:58.573985105Z","hostname":"{{ip-10-150-1-55.us-west-2.compute.internal"","msg"pid":"msg":"4759:"Flush timeout: 86400",","looperv[1] 5c1ab357-2b46-4e37-877a-5089e53cee0e looper connected""":":1,"","vupstairs0"":,"0:,"name1"name"::"}"cruciblecrucible"","
,level"":level":3030,"time":"2023-09-22T23:14:58.574056303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",","timepid":"4759:","upstairs":2023-09-22T23:14:58.574056445Z"1},
"hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid"":msg"4759:","looper":"1"[0] 04482c5c-f564-49f3-897c-51c4de58239a looper connected",","upstairs":v"1:0},
"name":"crucible","level":30,"time":"2023-09-22T23:14:58.574115845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:36921 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.574159066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] 04482c5c-f564-49f3-897c-51c4de58239a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.574199714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"1","upstairs":1}
{"msg":"[1] Proc runs for 127.0.0.1:37306 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.574242613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] 04482c5c-f564-49f3-897c-51c4de58239a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.574291754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"2","upstairs":1}
{"msg":"[2] Proc runs for 127.0.0.1:55683 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.574335298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] 28a12560-c07c-41d3-8d35-7e06b0c337aa looper connected","v":0,"name":"crucible","level":30{"msg":"[1] Proc runs for 127.0.0.1:47390 in state New","v":,"0time",:""name":"2023-09-22T23:14:58.574424762Z"crucible",,""hostname":"level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"2","upstairs":1}The guest has requested activation
,"time":"2023-09-22T23:14:58.5744514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] 5c1ab357-2b46-4e37-877a-5089e53cee0e looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.57451087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"2","upstairs":1}
{"msg":"[2] Proc runs for 127.0.0.1:35000 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.574558229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
The guest has requested activation
{"msg":"[1] Proc runs for 127.0.0.1:62630 in state New","v":0,"name":"crucible","level":30The guest has requested activation
,"time":"2023-09-22T23:14:58.5746426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] 85682a66-27fd-403c-bf22-d04b4ff66805 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.574689154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"
{"{msg":""[2] Proc runs for 127.0.0.1:61141 in state New"msg":","v":0,"5c1ab357-2b46-4e37-877a-5089e53cee0e active request set"name":","crucible"v":,0","levelname":"":crucible"30,"level":30,,""time"time"::""2023-09-22T23:14:58.574758151Z2023-09-22T23:14:58.574759835Z"",","hostname":hostname":""ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid"::47594759,,""upstairsupstairs""::11}
}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa looper connected","v":0,"name":"crucible","level":30looper":"2","upstairs":1}
,"time":"{2023-09-22T23:14:58.574833972Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":[2] Proc runs for 127.0.0.1:41929 in state New"4759,"v,"":looper0",The guest has requested activation
":"name"0":","crucible"upstairs",:"1level":}
30{"msg":"[0] Proc runs for 127.0.0.1:36984 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.574884193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,","upstairs"time"::"1}
2023-09-22T23:14:58.574898126Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":"4759msg":","upstairs":185682a66-27fd-403c-bf22-d04b4ff66805 active request set"}
,"v":0,"name{":"crucible",""msg"level":":3028a12560-c07c-41d3-8d35-7e06b0c337aa active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.57494455Z","hostname",:""time":"ip-10-150-1-55.us-west-2.compute.internal",2023-09-22T23:14:58.574952288Z"","pid"hostname"::"4759,"ip-10-150-1-55.us-west-2.compute.internalupstairs":"1,"}pid
":4759,"upstairs":1}
{"msg":"04482c5c-f564-49f3-897c-51c4de58239a active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575027733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 04482c5c-f564-49f3-897c-51c4de58239a, session_id: cc12d06e-bd83-45fe-987f-779d8c6339ec, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30{"msg":","time":"2023-09-22T23:14:58.575079275Z"negotiate packet HereIAm { version: 4, upstairs_id: 5c1ab357-2b46-4e37-877a-5089e53cee0e, session_id: fcec4285-eaa1-46a4-819c-725bdeb5d523, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","hostname,"":v"":0,"ip-10-150-1-55.us-west-2.compute.internal"name,"":pid"":crucible"4759,","level":downstairs30":1}
{"msg":","time":"2023-09-22T23:14:58.575112938Z","hostname":"negotiate packet HereIAm { version: 4, upstairs_id: 04482c5c-f564-49f3-897c-51c4de58239a, session_id: cc12d06e-bd83-45fe-987f-779d8c6339ec, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }"ip-10-150-1-55.us-west-2.compute.internal",,""pid"v"::47590,"name,"":"downstairs"crucible":,"1level":}30
{"msg":","time":"2023-09-22T23:14:58.57515318Z","hostname":"negotiate packet HereIAm { version: 4, upstairs_id: 5c1ab357-2b46-4e37-877a-5089e53cee0e, session_id: fcec4285-eaa1-46a4-819c-725bdeb5d523, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }"ip-10-150-1-55.us-west-2.compute.internal",",v""pid"::04759,"name,"":"downstairs":crucible2"},"
level":30,"time":"2023-09-22T23:14:58.575192243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":2}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 5c1ab357-2b46-4e37-877a-5089e53cee0e, session_id: fcec4285-eaa1-46a4-819c-725bdeb5d523, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575244474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":3}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 04482c5c-f564-49f3-897c-51c4de58239a, session_id: cc12d06e-bd83-45fe-987f-779d8c6339ec, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575313198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":3}
{"msg":"[0] 04482c5c-f564-49f3-897c-51c4de58239a (cc12d06e-bd83-45fe-987f-779d8c6339ec) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575367641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575409123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] client is_active_req TRUE, promote! session cc12d06e-bd83-45fe-987f-779d8c6339ec","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575452431Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] 04482c5c-f564-49f3-897c-51c4de58239a (cc12d06e-bd83-45fe-987f-779d8c6339ec) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575496585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575538655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] client is_active_req TRUE, promote! session cc12d06e-bd83-45fe-987f-779d8c6339ec","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575581714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] 04482c5c-f564-49f3-897c-51c4de58239a (cc12d06e-bd83-45fe-987f-779d8c6339ec) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575627381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575669589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] client is_active_req TRUE, promote! session cc12d06e-bd83-45fe-987f-779d8c6339ec","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575711077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: 04482c5c-f564-49f3-897c-51c4de58239a, session_id: cc12d06e-bd83-45fe-987f-779d8c6339ec, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575757682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: 04482c5c-f564-49f3-897c-51c4de58239a, session_id: cc12d06e-bd83-45fe-987f-779d8c6339ec, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575804198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":2}
{"msg":"negotiate packet PromoteToActive { upstairs_id: 04482c5c-f564-49f3-897c-51c4de58239a, session_id: cc12d06e-bd83-45fe-987f-779d8c6339ec, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.57585122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":3}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 28a12560-c07c-41d3-8d35-7e06b0c337aa, session_id: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575923262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 28a12560-c07c-41d3-8d35-7e06b0c337aa, session_id: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.575976358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":2}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 28a12560-c07c-41d3-8d35-7e06b0c337aa, session_id: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.576027274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":3}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.57608157Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"{2023-09-22T23:14:58.576124225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] client is_active_req TRUE, promote! session 7f1ba66d-4c7a-4444-a654-4b5bfde1244a","v":0,"name":"crucible","level":30"msg":","time":"[0] 5c1ab357-2b46-4e37-877a-5089e53cee0e (fcec4285-eaa1-46a4-819c-725bdeb5d523) New New New ds_transition to WaitActive"2023-09-22T23:14:58.576165198Z",",v"":hostname":"0,"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible",","pid"level:"4759:30,"upstairs":1}
{"msg":"[1] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) WaitActive New New ds_transition to WaitActive","v":0,"name":","crucible"time",":"level":302023-09-22T23:14:58.576196202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,"time":"2023-09-22T23:14:58.576222332Z","{hostname":""ip-10-150-1-55.us-west-2.compute.internal"msg":","pid":4759[0] Transition from New to WaitActive",","upstairs":v"1:}0
,"name":"crucible","level{":30"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.576265344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,"time":"2023-09-22T23:14:58.576275563Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,[0] client is_active_req TRUE, promote! session fcec4285-eaa1-46a4-819c-725bdeb5d523"","upstairs"v"::10,}"
name":"crucible","level":{30"msg":"[1] client is_active_req TRUE, promote! session 7f1ba66d-4c7a-4444-a654-4b5bfde1244a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.576319558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,"time":"{2023-09-22T23:14:58.576332234Z",""hostnamemsg"":":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1[1] 5c1ab357-2b46-4e37-877a-5089e53cee0e (fcec4285-eaa1-46a4-819c-725bdeb5d523) WaitActive New New ds_transition to WaitActive"}
,"v":0,"name":{"crucible",""levelmsg":"30:"[2] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.576376652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,"time":"{2023-09-22T23:14:58.576388258Z",""hostname":"msg":"ip-10-150-1-55.us-west-2.compute.internal",[1] Transition from New to WaitActive""pid":,"4759v":,0","upstairs"name:"1:"}
crucible","level":30{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.576430139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,"time":"{2023-09-22T23:14:58.576440847Z",""hostnamemsg"":":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] client is_active_req TRUE, promote! session fcec4285-eaa1-46a4-819c-725bdeb5d523",","upstairs"v"::10,}"
name":"crucible","level"{:30"msg":"[2] client is_active_req TRUE, promote! session 7f1ba66d-4c7a-4444-a654-4b5bfde1244a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.5764846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,"time":"{2023-09-22T23:14:58.576496819Z"","msg":hostname":""ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1[2] 5c1ab357-2b46-4e37-877a-5089e53cee0e (fcec4285-eaa1-46a4-819c-725bdeb5d523) WaitActive WaitActive New ds_transition to WaitActive}"
,"v":0,"name":"{crucible","level":30"msg":"negotiate packet PromoteToActive { upstairs_id: 28a12560-c07c-41d3-8d35-7e06b0c337aa, session_id: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a, gen: 1 }","v":0,"name":"crucible",",level"":time":30"2023-09-22T23:14:58.576541556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{,"time"":"msg":"2023-09-22T23:14:58.576558314Z",[2] Transition from New to WaitActive"","hostname":v"":0,"name"ip-10-150-1-55.us-west-2.compute.internal":","crucible"pid,"":level"4759:30,"downstairs":1}
{,""time"msg":":"2023-09-22T23:14:58.576594265Z","hostname":"negotiate packet PromoteToActive { upstairs_id: 28a12560-c07c-41d3-8d35-7e06b0c337aa, session_id: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a, gen: 1 }ip-10-150-1-55.us-west-2.compute.internal"",",pid":"4759v":,"0upstairs,""name:"1:"}
crucible","level":{30"msg":"[2] client is_active_req TRUE, promote! session fcec4285-eaa1-46a4-819c-725bdeb5d523","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.576634135Z","hostname":",ip-10-150-1-55.us-west-2.compute.internal""time,":""pid":47592023-09-22T23:14:58.576645723Z",","hostname":"downstairs":2}ip-10-150-1-55.us-west-2.compute.internal"
,"pid":4759,"upstairs"{:1}
"msg":"{"msg":"negotiate packet PromoteToActive { upstairs_id: 28a12560-c07c-41d3-8d35-7e06b0c337aa, session_id: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a, gen: 1 }","v":0,"name":"crucible"negotiate packet PromoteToActive { upstairs_id: 5c1ab357-2b46-4e37-877a-5089e53cee0e, session_id: fcec4285-eaa1-46a4-819c-725bdeb5d523, gen: 1 },"","level"v"::300,"name":"crucible","level":30,",time"":"time":"2023-09-22T23:14:58.57670099Z"2023-09-22T23:14:58.576704972Z",",hostname"":hostname"":"ip-10-150-1-55.us-west-2.compute.internal",ip-10-150-1-55.us-west-2.compute.internal"",pid""pid"::47594759,","downstairs":downstairs"1:}3
}
{"msg":"{"msg":"negotiate packet RegionInfoPlease","v":negotiate packet PromoteToActive { upstairs_id: 5c1ab357-2b46-4e37-877a-5089e53cee0e, session_id: fcec4285-eaa1-46a4-819c-725bdeb5d523, gen: 1 }0",",name"":"v":crucible0",","name"level"::30"crucible","level":30,"time":","2023-09-22T23:14:58.576768052Z"time":","hostname":2023-09-22T23:14:58.576770695Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,","pid":downstairs4759":1,"}
downstairs":2}
{"msg"{:""negotiate packet RegionInfoPlease"msg",":"v":0,"name":"crucible","level":30negotiate packet PromoteToActive { upstairs_id: 5c1ab357-2b46-4e37-877a-5089e53cee0e, session_id: fcec4285-eaa1-46a4-819c-725bdeb5d523, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.576819735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid,"":time4759":","downstairs":2023-09-22T23:14:58.576827899Z"2,"}
hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid"":msg4759":","downstairs":negotiate packet RegionInfoPlease"3,"}v"
:0,"name":"{crucible","level":"30msg":"negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.576874066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"time",:""downstairs":2023-09-22T23:14:58.57688217Z"3,"}
hostname":"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759,""msg":downstairs"":1}
[0] downstairs client at 127.0.0.1:36984 has UUID 8dfe8526-b15f-436d-8c37-dad574df0099","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.57692579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 8dfe8526-b15f-436d-8c37-dad574df0099, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.576974457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"28a12560-c07c-41d3-8d35-7e06b0c337aa WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577017702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] downstairs client at 127.0.0.1:46704 has UUID 62fdae0f-000a-4ad8-95c6-010549b306c9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577064597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 62fdae0f-000a-4ad8-95c6-010549b306c9, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577113792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"28a12560-c07c-41d3-8d35-7e06b0c337aa WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577154824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] downstairs client at 127.0.0.1:61141 has UUID a3b7c7a7-d0a0-4a01-b35d-0e669d8733e5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577204699Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: a3b7c7a7-d0a0-4a01-b35d-0e669d8733e5, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577250952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"28a12560-c07c-41d3-8d35-7e06b0c337aa WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577288566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577329553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577369386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":2}
{"msg":"negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577409851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":3}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577456626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577496787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:58.577538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.57757817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577626212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577667205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:58.577707678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577748227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577800409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577843686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:58.577885383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.577926208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 127.0.0.1:36984 task reports connection:true","v":0,"name":"crucible","level":30,"The guest has finished waiting for activation
time":"2023-09-22T23:14:58.577966987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"28a12560-c07c-41d3-8d35-7e06b0c337aa WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.578017124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.578059268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.578117737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0]R  dirty: [false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.578160705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.578203545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{{"msg":""[1]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]",msg"":v":"0,"name":"negotiate packet RegionInfoPlease"crucible",","level"v"::300,"name":"crucible","level":30{","msgtime":"":"2023-09-22T23:14:58.578253727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":,"1negotiate packet HereIAm { version: 4, upstairs_id: 85682a66-27fd-403c-bf22-d04b4ff66805, session_id: 5d828379-bf79-455f-9270-e82849f9ecba, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }"time"}:
","v":2023-09-22T23:14:58.578259533Z0"{,","name":""msg"crucible":",hostname"":"level":[1]R  dirty: [false, false, false, false, false, false, false, false, false, false]"30ip-10-150-1-55.us-west-2.compute.internal,"","v"pid"::04759,"name":",crucible"","downstairs"level"::130}
,"time":"2023-09-22T23:14:58.57830414Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","negotiate packet RegionInfoPlease"pid",,":time4759""v,:""":0downstairs,""2023-09-22T23:14:58.578318907Zname":"1:,}"
"hostname":"crucible","{level"ip-10-150-1-55.us-west-2.compute.internal":"30msg":,""pid":4759,"upstairs":1}
negotiate packet HereIAm { version: 4, upstairs_id: 85682a66-27fd-403c-bf22-d04b4ff66805, session_id: 5d828379-bf79-455f-9270-e82849f9ecba, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"{name,"":"time":""msg":"2023-09-22T23:14:58.578366552Z"crucible",,"[2]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]hostname"":",""vlevel"":ip-10-150-1-55.us-west-2.compute.internal"30,:"0pid":,"4759name":","crucible"downstairs",":level":230},"
time":"2023-09-22T23:14:58.578400272Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":negotiate packet RegionInfoPlease4759",",downstairs":,""2v"}time":
":0,"2023-09-22T23:14:58.578411249Z"name{",:"""hostname"msg":"crucible":","ip-10-150-1-55.us-west-2.compute.internal"level",":pid"30:4759,"upstairs":negotiate packet HereIAm { version: 4, upstairs_id: 85682a66-27fd-403c-bf22-d04b4ff66805, session_id: 5d828379-bf79-455f-9270-e82849f9ecba, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }"1,"}v
":0,"name,""{:"time"crucible:"""msg,""2023-09-22T23:14:58.578460689Z"level":,:"30"hostname":"[2]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":ip-10-150-1-55.us-west-2.compute.internal0",,""pid"name"::"4759crucible",,""level"downstairs:,""30:time"3:"}
2023-09-22T23:14:58.578487337Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal",""pid":msg"4759:",","time":"downstairs":[0] downstairs client at 127.0.0.1:36921 has UUID d057f0d7-1fef-421b-b789-cb2746bfdb262023-09-22T23:14:58.578504702Z"3",}"
v",:"0hostname,{"":"name":""crucibleip-10-150-1-55.us-west-2.compute.internal"msg"",",:"pid"":level"4759:30,[0] 85682a66-27fd-403c-bf22-d04b4ff66805 (5d828379-bf79-455f-9270-e82849f9ecba) New New New ds_transition to WaitActive"",upstairs"":v"1:}0
,"name":","crucible"time":{","level":2023-09-22T23:14:58.578550382Z""msg"30,:""hostname"{:"[2]R  dirty: [false, false, false, false, false, false, false, false, false, false]"ip-10-150-1-55.us-west-2.compute.internal",","msgv""pid"":::,""04759negotiate packet RegionInfoPleasetime"",,:""v",2023-09-22T23:14:58.578571948Z"":,0","name"upstairs"hostname""name:"":"::"1ip-10-150-1-55.us-west-2.compute.internalcrucible"",",pid"crucible":"4759level",:"}upstairs30"
,:"1level{}"
":msg30",{:"""msgtime""::""2023-09-22T23:14:58.578621037Z[0] Transition from New to WaitActive"",,""v":hostname"0:","name":"crucibleip-10-150-1-55.us-west-2.compute.internal"",,""levelpid"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: d057f0d7-1fef-421b-b789-cb2746bfdb26, encrypted: false, database_read_version: 1, database_write_version: 1 }""::304759,"v",":downstairs0",:",2,"}"name":"time
":"time"crucible{2023-09-22T23:14:58.578649287Z:""msg,"":"hostname"":negotiate packet RegionInfoPlease""",",vip-10-150-1-55.us-west-2.compute.internal"":,""0pid,"":name4759"2023-09-22T23:14:58.578635805Z"level",,:"30":"upstairscrucible"":,1"}hostname"level
"::{30""msg":"ip-10-150-1-55.us-west-2.compute.internal",,""pid":[0] client is_active_req TRUE, promote! session 5d828379-bf79-455f-9270-e82849f9ecba"4759time":,"",,""vtime""::0",upstairs""2023-09-22T23:14:58.578703489Z:2023-09-22T23:14:58.578693337Z"1name"",:""hostname"crucible:""},
,""ip-10-150-1-55.us-west-2.compute.internalhostname""level,"":pid30{:"":4759,""ip-10-150-1-55.us-west-2.compute.internal"downstairs"msg,:,3"}time
"":{"pid"""2023-09-22T23:14:58.578740164Zmsg"::",""4759hostname:"":"Max found gen is 1",","upstairs"v":ip-10-150-1-55.us-west-2.compute.internal[0] downstairs client at 127.0.0.1:58766 has UUID f460c951-f672-4bce-b7d4-32f439e7d15b:"",,""v1"pid}:"0
,:"4759name":","crucibleupstairs"",:"1level{0"}:
30",msg""{name"::"""crucible,msg""time:"""04482c5c-f564-49f3-897c-51c4de58239a WaitActive WaitActive WaitActive",",level"":v":[1] 85682a66-27fd-403c-bf22-d04b4ff66805 (5d828379-bf79-455f-9270-e82849f9ecba) WaitActive New New ds_transition to WaitActive:"300,"","2023-09-22T23:14:58.578797044Zv"",:"0name":"hostname,"":name"":"crucible"ip-10-150-1-55.us-west-2.compute.internalcrucible"",,""pid"level:"4759:,30,""level":upstairs",":301time},
"":"{time":"2023-09-22T23:14:58.578819025Z"2023-09-22T23:14:58.578835342Z""msg",:""hostname",,":""time":hostname"ip-10-150-1-55.us-west-2.compute.internal":",""[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: f460c951-f672-4bce-b7d4-32f439e7d15b, encrypted: false, database_read_version: 1, database_write_version: 1 }pid"",:"4759v",:"0upstairs,"":2023-09-22T23:14:58.578842192Z1name"}:
""crucible{",",msg""level:"":ip-10-150-1-55.us-west-2.compute.internal30"[1] Transition from New to WaitActivehostname""":,,""v":"0,,""timeip-10-150-1-55.us-west-2.compute.internal"pid"name:"",":pid"4759:":,"2023-09-22T23:14:58.578879784Z"crucible",,""hostname"level:"4759":"30,upstairs""ip-10-150-1-55.us-west-2.compute.internal:"1,upstairs":}1
},
"{time":"2023-09-22T23:14:58.578910648Z"{,""msg":hostname""":"msg"":ip-10-150-1-55.us-west-2.compute.internal""pid,"":Generation requested: 1 >= found:1"4759pid[1] downstairs client at 127.0.0.1:37306 has UUID 62157a5b-b69c-4edc-9cca-1fe4a003d85b",,"","upstairs:"4759:"1,"}v":v"0upstairs
"::10{},""
msg":,name""{"name":":"msg5c1ab357-2b46-4e37-877a-5089e53cee0e WaitActive WaitActive WaitActive"":",""crucible"vcrucible,"[1] client is_active_req TRUE, promote! session 5d828379-bf79-455f-9270-e82849f9ecba"test control::test::test_crucible_control_openapi ... ,"ok
levellevel"":"",:"0v,"":name0:,"":name""3030crucible:"","crucible"level,"":level30":30,"time":",,""time2023-09-22T23:14:58.579015432Z"",:time""",time"":2023-09-22T23:14:58.579018891Z"hostname":test impacted_blocks::test::empty_impacted_blocks_never_conflict ... ":ip-10-150-1-55.us-west-2.compute.internal"ok
"","2023-09-22T23:14:58.579018446Z,""2023-09-22T23:14:58.579016116Z,hostname"""hostname:"",:"pid"":ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidhostname"4759:""pid,:"4759:"ip-10-150-1-55.us-west-2.compute.internal"upstairs,",4759","upstairs"":upstairs1"}:
1:pid"{}1
":msg{}4759"test impacted_blocks::test::empty_contains_nothing ... msg""::""
ok
[1] downstairs client at 127.0.0.1:47390 has UUID 03030c34-ede9-4da3-b23e-b6a582013eba[2] 85682a66-27fd-403c-bf22-d04b4ff66805 (5d828379-bf79-455f-9270-e82849f9ecba) WaitActive WaitActive New ds_transition to WaitActive"",,""vv""::00,,""{namename""::""cruciblecrucible"",,""levellevel""::3030"msg":"Next flush: 1","v":0,"name":"crucible",",,""level"timetime""::"":302023-09-22T23:14:58.579132776Z2023-09-22T23:14:58.579132728Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759,,""upstairsupstairs""::11}}

{{"msg":""msg":,""time":"[2] Transition from New to WaitActive",2023-09-22T23:14:58.579149971Z""v":,0",[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 03030c34-ede9-4da3-b23e-b6a582013eba, encrypted: false, database_read_version: 1, database_write_version: 1 }""name",:""v"crucible:"0,,""levelname"hostname":":":30"crucible"ip-10-150-1-55.us-west-2.compute.internal",","level"pid"::304759,",upstairs"":time":1"}
,2023-09-22T23:14:58.57918566Z""time",:""hostname":"2023-09-22T23:14:58.579192756Z"{,"ip-10-150-1-55.us-west-2.compute.internal"hostname,"":"pid":4759",ip-10-150-1-55.us-west-2.compute.internal""msgupstairs,"":pid1""}:
4759:","{All extents match"upstairs"",:msg1"}:
""v":{0,""msg[2] client is_active_req TRUE, promote! session 5d828379-bf79-455f-9270-e82849f9ecba"":,"name"":v5c1ab357-2b46-4e37-877a-5089e53cee0e WaitActive WaitActive WaitActive"":,0",v"":name0",:""name"crucible:""",crucible""level,"":level30":crucible"30,"level":30,"time,"":"time":"2023-09-22T23:14:58.579248615Z"2023-09-22T23:14:58.579250601Z,"","hostname":"hostname":"ip-10-150-1-55.us-west-2.compute.internal",ip-10-150-1-55.us-west-2.compute.internal""pid,"":pid4759":4759,","upstairs":upstairs1","}:
1time"}{
":"msg{":"2023-09-22T23:14:58.579253032Z""msg":,""hostname":"negotiate packet PromoteToActive { upstairs_id: 85682a66-27fd-403c-bf22-d04b4ff66805, session_id: 5d828379-bf79-455f-9270-e82849f9ecba, gen: 1 }"ip-10-150-1-55.us-west-2.compute.internal"[2] downstairs client at 127.0.0.1:35000 has UUID 27b6e795-c830-4cf3-a348-4b6152c16b91,"",v""v"::00,,""name","name:"":pid""crucible"crucible":,,""4759levellevel""::3030,"upstairs":1}
,,""timetime"":":"{2023-09-22T23:14:58.579302586Z"2023-09-22T23:14:58.579302622Z",","hostname":hostname""":msg"ip-10-150-1-55.us-west-2.compute.internal"":","ip-10-150-1-55.us-west-2.compute.internal"pid",No downstairs repair required":"4759pid,,""":upstairs4759"v",:"1:}downstairs
"0:1{,"}name"
msg""::{""crucible""msg":,""level":30[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 27b6e795-c830-4cf3-a348-4b6152c16b91, encrypted: false, database_read_version: 1, database_write_version: 1 }"negotiate packet PromoteToActive { upstairs_id: 85682a66-27fd-403c-bf22-d04b4ff66805, session_id: 5d828379-bf79-455f-9270-e82849f9ecba, gen: 1 }",",v""v:":00,,""namename""::""cruciblecrucible"",,""level"level:":3030,"time":"2023-09-22T23:14:58.579356682Z","hostname":",,""timetime""::""ip-10-150-1-55.us-west-2.compute.internal","pid2023-09-22T23:14:58.579367984Z2023-09-22T23:14:58.579368016Z""":,,"4759"hostname":hostname"",":ip-10-150-1-55.us-west-2.compute.internal""upstairs",ip-10-150-1-55.us-west-2.compute.internal""pid,"":pid":47594759:,,""1upstairsdownstairs""::12}}}

{
"{msg":""msg":"{5c1ab357-2b46-4e37-877a-5089e53cee0e WaitActive WaitActive WaitActive","v":"0msg,negotiate packet PromoteToActive { upstairs_id: 85682a66-27fd-403c-bf22-d04b4ff66805, session_id: 5d828379-bf79-455f-9270-e82849f9ecba, gen: 1 }""name",:""v":crucible0",,""name"level:"":"30crucible":,""level":30No initial repair work was required","v":0,"name":"crucible","level,":"30time,"":"time":"2023-09-22T23:14:58.579431947Z"2023-09-22T23:14:58.57943562Z,"","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal",ip-10-150-1-55.us-west-2.compute.internal""pid,"":pid4759":4759,","upstairs":downstairs1":}3
}
{,"time""{:""msg"msg:"":"2023-09-22T23:14:58.579444043Z","negotiate packet ExtentVersionsPleasenegotiate packet RegionInfoPlease"",",v"":v0hostname":,"":name0",:""name"crucible:"",""crucible"level,"":30level":ip-10-150-1-55.us-west-2.compute.internal30","pid":4759,"upstairs":1}
The guest has finished waiting for activation
{,,""timetime""::"""msg":"2023-09-22T23:14:58.579478924Z2023-09-22T23:14:58.579480369Z"",,"Set Downstairs and Upstairs active"hostname"":hostname"",":ip-10-150-1-55.us-west-2.compute.internal"",v""ip-10-150-1-55.us-west-2.compute.internal"pid,"":pid4759":,4759:",0downstairs"",downstairs:"1:"}1
}name":"{
"cruciblemsg{":""",msg"":negotiate packet ExtentVersionsPlease""level,"negotiate packet RegionInfoPlease""v",:"0v,"":name0",:""name"crucible:"","crucible"level,"":level30"::3030,,"","time"time:"":"upstairs"2023-09-22T23:14:58.579546302Z"2023-09-22T23:14:58.579548053Z",:","hostname"hostname"::""1,"time"ip-10-150-1-55.us-west-2.compute.internal}:"ip-10-150-1-55.us-west-2.compute.internal,"",pid""pid:"4759:
,4759""downstairs,"":2downstairs":}2
}{2023-09-22T23:14:58.579548886Z"
{,"""{msghostname":"""msg:":""msg"ip-10-150-1-55.us-west-2.compute.internal"negotiate packet RegionInfoPleasenegotiate packet ExtentVersionsPlease"",,,""vv""::00,,":pid":"""name"name"::""4759cruciblecrucible"",,"",levellevel""::3030"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 62157a5b-b69c-4edc-9cca-1fe4a003d85b, encrypted: false, database_read_version: 1, database_write_version: 1 }"upstairs":1,"}v
",,""timetime""::""{:2023-09-22T23:14:58.579620624Z2023-09-22T23:14:58.579620936Z"",0"",msg"hostname"hostname:"":,""name":ip-10-150-1-55.us-west-2.compute.internal"":""ip-10-150-1-55.us-west-2.compute.internal,"",pid""pid:"4759:28a12560-c07c-41d3-8d35-7e06b0c337aa is now active with session: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a4759"crucible",,"",downstairsdownstairs,""::33}}

{""{"v""msglevel""msg:"":"::300,"[0] downstairs client at 127.0.0.1:39040 has UUID b0da680b-dcef-4405-a0bc-9a419607b615"name",:"[0] 5c1ab357-2b46-4e37-877a-5089e53cee0e (fcec4285-eaa1-46a4-819c-725bdeb5d523) WaitActive WaitActive WaitActive ds_transition to WaitQuorumv"",:"0v,"""name:"0:,""cruciblename""crucible",:","levelcrucible"":,30""level"level"::3030,"time":"2023-09-22T23:14:58.579684613Z",","hostname":time":",""time2023-09-22T23:14:58.579702252Z"":","hostname2023-09-22T23:14:58.579706536Z"":ip-10-150-1-55.us-west-2.compute.internal,"""hostname",ip-10-150-1-55.us-west-2.compute.internal"":,"",pid"ip-10-150-1-55.us-west-2.compute.internalpid"",:"4759pid","":upstairs:"4759time"::,1"}"
upstairs2023-09-22T23:14:58.579706448Z""{:,"1"4759}msg
","hostname"upstairs{:"""msg::":"1"}
ip-10-150-1-55.us-west-2.compute.internal"[0] Transition from WaitActive to WaitQuorum[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: b0da680b-dcef-4405-a0bc-9a419607b615, encrypted: false, database_read_version: 1, database_write_version: 1 }"",,""vv""::00,,""name"name:""{:crucible"",",crucible""level,"":level30":pid30"":msg4759":","upstairs":1}04482c5c-f564-49f3-897c-51c4de58239a WaitActive WaitActive WaitActive"
,,,""time"time:"":""2023-09-22T23:14:58.579795946Z{v""2023-09-22T23:14:58.579794215Z",,"":"0hostnamehostname""::"","msg"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:"4759::4759,","name"upstairs"":upstairs28a12560-c07c-41d3-8d35-7e06b0c337aa Set Active after no repair""1:}1
}:,{
""msg{v"":""msg:"":0","crucible[0] new RM replaced this: None85682a66-27fd-403c-bf22-d04b4ff66805 WaitActive WaitActive WaitActive""",,",name":"""vv""::00,,""namename""::level""":crucible"cruciblecrucible"",,""levellevel""::3040,"30level":30,,""timetime""::""2023-09-22T23:14:58.579888065Z2023-09-22T23:14:58.579887789Z"",,""hostnamehostname""::"","ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",",pid""pid"::47594759time",:""time":",,"2023-09-22T23:14:58.579889617Z"2023-09-22T23:14:58.579891213Z",upstairs"":upstairs1"}
:,""{hostname""1msg}"
:"hostname":{[0] Starts reconcile loop"""msg,"":v"":ip-10-150-1-55.us-west-2.compute.internal":,"0[1] downstairs client at 127.0.0.1:62630 has UUID 135d3cb9-fa87-4418-b94e-747b9c756857",,"""vname""::pid"0",crucible"":,name""level:"":ip-10-150-1-55.us-west-2.compute.internalcrucible30"4759,"",,"level"upstairs"":pid,:"30"1:time"4759}:
,"",time"2023-09-22T23:14:58.579963189Z:"",{"2023-09-22T23:14:58.579974259Z""upstairs",":msg1"":"hostnamehostname""::""Notify all downstairs, region set compare is done."}ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""
pidpid""::47594759,,,""upstairs"upstairs:"{1"}:
1v""{msg"}"
msg":"::{"0","msg"name"::""crucible"[2] downstairs client at 127.0.0.1:55683 has UUID b50373f0-d859-4c68-a8df-7a98765c5610,""[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 135d3cb9-fa87-4418-b94e-747b9c756857, encrypted: false, database_read_version: 1, database_write_version: 1 }"level,[1] 5c1ab357-2b46-4e37-877a-5089e53cee0e (fcec4285-eaa1-46a4-819c-725bdeb5d523) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum"",v,"":v30"""v:"0:,0",name"":name:"0:"",cruciblecrucible"",,"""levellevel""::3030name":"crucible","level":30,The guest has finished waiting for activation
",,""time"time:"":"time":"2023-09-22T23:14:58.580066485Z"2023-09-22T23:14:58.580066157Z",",2023-09-22T23:14:58.580052974Z"hostname"":"hostname",",":ip-10-150-1-55.us-west-2.compute.internal""time"hostname":"ip-10-150-1-55.us-west-2.compute.internal,"",pid"":"ip-10-150-1-55.us-west-2.compute.internal"pid:"4759:2023-09-22T23:14:58.580070567Z"4759,,",,"""hostname":"upstairsupstairs""::pid11}}

ip-10-150-1-55.us-west-2.compute.internal"{{,"":"4759pid"msg""msg:"":":,"4759upstairs85682a66-27fd-403c-bf22-d04b4ff66805 WaitActive WaitActive WaitActive[1] Transition from WaitActive to WaitQuorum""",,""vv""::00,,"","namename""::"":upstairs":cruciblecrucible"",,""11levellevel"}:
}30"
{:30","msg"time"::""Set check for repair"2023-09-22T23:14:58.580163166Z,"",time,"":""hostname"2023-09-22T23:14:58.580169102Z:"",v""ip-10-150-1-55.us-west-2.compute.internal"hostname,"":pid""::4759ip-10-150-1-55.us-west-2.compute.internal",,""0upstairspid,"""::14759}name
,"":"{upstairs"crucible{"":,1msg"}"
:""level"[1] new RM replaced this: None{":",msg"":"v"msg"30::[2] downstairs client at 127.0.0.1:41929 has UUID 533946c5-94b5-4106-8c49-f677757735e20",,""namev""::"0",crucible""name,"":"level":crucible"40,"level":30[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: b50373f0-d859-4c68-a8df-7a98765c5610, encrypted: false, database_read_version: 1, database_write_version: 1 }",",time":"",,""timetime2023-09-22T23:14:58.580219706Z""":",:2023-09-22T23:14:58.58023078Z""",2023-09-22T23:14:58.580229367Z""hostname",:""hostnamehostnamev"":":ip-10-150-1-55.us-west-2.compute.internal""0:,""ip-10-150-1-55.us-west-2.compute.internal"pid",:ip-10-150-1-55.us-west-2.compute.internal4759",",,""pid"upstairs:"":pid"14759}:name,
""upstairs:4759"{:",""1msg}"
:upstairs"{":crucible"1",}"msg"
level":[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 533946c5-94b5-4106-8c49-f677757735e2, encrypted: false, database_read_version: 1, database_write_version: 1 }"":,[1] Starts reconcile loop""v,""v:"0{30,:"0name,"":"name":crucible""",cruciblemsg"""level,"":level30":":30[1] 127.0.0.1:46704 task reports connection:true","v":0,"name":"crucible",","time"level,,""time"time:"":""2023-09-22T23:14:58.580328228Z":,2023-09-22T23:14:58.580329805Z""hostname,"":30hostname"":":ip-10-150-1-55.us-west-2.compute.internal"","2023-09-22T23:14:58.580319951Z"pidip-10-150-1-55.us-west-2.compute.internal"":,4759",,pid"""upstairs:"4759:hostname1,}"":upstairs
":"1{},
"ip-10-150-1-55.us-west-2.compute.internal"{msg"time":""":msg2023-09-22T23:14:58.580351387Z","","":pid"hostname":":"4759ip-10-150-1-55.us-west-2.compute.internal"85682a66-27fd-403c-bf22-d04b4ff66805 WaitActive WaitActive WaitActive",,""pid,"":[2] 5c1ab357-2b46-4e37-877a-5089e53cee0e (fcec4285-eaa1-46a4-819c-725bdeb5d523) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorumv"",:4759upstairs""0v,"":name0",:",:name""crucible:""1"crucible,"",level":}30"upstairs"
level"::130,"{}time":
"","2023-09-22T23:14:58.580416985Z"time,"":hostname"":"msg":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.580425677Z"",,""{hostnamepid""::"475904482c5c-f564-49f3-897c-51c4de58239a WaitActive WaitActive WaitActive,""ip-10-150-1-55.us-west-2.compute.internal""upstairs,","msg":pid""::14759}v",
":"0{upstairs":"1msg"}:
",28a12560-c07c-41d3-8d35-7e06b0c337aa Active Active Active{negotiate packet ExtentVersionsPlease"""",msg""v:"":,0"name",[2] Transition from WaitActive to WaitQuorum""name",:""v"crucible":,0v",""level:name""::30"0:crucible",",""name"crucible"level,":",":time30"crucible"",":level"":level":3030,2023-09-22T23:14:58.580495299Z""time",:""hostname":"2023-09-22T23:14:58.580508202Z","hostnameip-10-150-1-55.us-west-2.compute.internal"":","pid":4759ip-10-150-1-55.us-west-2.compute.internal,"","downstairspid""::14759}
,"upstairs":1{}
",msg{":",""time""negotiate packet ExtentVersionsPleasemsg"",:""v":"time"[2] new RM replaced this: None::"2023-09-22T23:14:58.580517127Z0",,""namev""::"2023-09-22T23:14:58.580517973Z"0crucible",",,"",hostname""name"":level:"":crucible30"","hostnamelevel"ip-10-150-1-55.us-west-2.compute.internal":,40":""pid":,"4759time":ip-10-150-1-55.us-west-2.compute.internal"",,"2023-09-22T23:14:58.580569449Z""time",:"","hostname2023-09-22T23:14:58.580575631Z"":,""pid"hostnameip-10-150-1-55.us-west-2.compute.internal"",::upstairs"4759:"","1pidip-10-150-1-55.us-west-2.compute.internal"":,4759"upstairs"}:pid,"":
downstairs4759":12,"}{
upstairs"{"msg:"1"}msg:}""
:"negotiate packet ExtentVersionsPlease"
thread 'impacted_blocks::test::extent_from_offset_panics_when_num_blocks_outside_region' panicked at 'assertion failed: offset.value as u128 + num_blocks.value as u128 <=\n    ddef.extent_count() as u128 * extent_size as u128', upstairs/src/impacted_blocks.rs:284:5
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
,negotiate packet ExtentVersionsPlease{""",msg""v:"":v{"[2] Starts reconcile loop0",:""0,msg""namev"":::,"""0crucible,""Set check for repair,name""level:"":"name":crucible30",",v""":level":crucible30"0,,""time"name"::"",,"2023-09-22T23:14:58.580667818Ztime"",:""crucible"hostname,"2023-09-22T23:14:58.580675655Z:""""level"level"ip-10-150-1-55.us-west-2.compute.internal,"","hostnamepid"":"::4759ip-10-150-1-55.us-west-2.compute.internal,""30downstairs,""::3pid"}:
475930,"{upstairs":1}
","{msg"time:""msg":"":"2023-09-22T23:14:58.580705528Z"[0] 127.0.0.1:58766 task reports connection:true",[0] 85682a66-27fd-403c-bf22-d04b4ff66805 (5d828379-bf79-455f-9270-e82849f9ecba) WaitActive WaitActive WaitActive ds_transition to WaitQuorum,"""v,"":v0",:"0name,"":name"":crucible"",,crucible""level",":level30":"hostnametime"":30":"2023-09-22T23:14:58.580717087Z"ip-10-150-1-55.us-west-2.compute.internal",,,""pid":,""hostname4759":"timetime""ip-10-150-1-55.us-west-2.compute.internal::"",2023-09-22T23:14:58.580748696Z"",,2023-09-22T23:14:58.580754128Z""",hostname""":hostnamepid"""::"ip-10-150-1-55.us-west-2.compute.internaldownstairs4759""ip-10-150-1-55.us-west-2.compute.internal,"",pid""pid:"4759:,:",4759"upstairs1"upstairs,""}upstairs::
11":}1}}

{{
"{msg""":msg""msg"{:":5c1ab357-2b46-4e37-877a-5089e53cee0e WaitQuorum WaitQuorum WaitQuorum"","v"[0] Transition from WaitActive to WaitQuorum:"0,,""v"name""::0",msg"negotiate packet ExtentVersionsPlease"crucible""name":,"":levelcrucible"":,30",level""":v"30:[2] 127.0.0.1:61141 task reports connection:true0","name",:,"""crucible"time,"":time"":",v""level"::02023-09-22T23:14:58.580845666Z2023-09-22T23:14:58.580851229Z"",,30,"""name":"hostnamehostnamecrucible"""::"","level":ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:"4759:304759,",,upstairs""":upstairs"1:}1
}time"
{:""{msg":""2023-09-22T23:14:58.580873878Z"msg":[0]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],""",time"[0] new RM replaced this: None""v,"":,:""0v,"":2023-09-22T23:14:58.580887613Z"0name,hostname,"""":hostname""::name""":crucible""crucibleip-10-150-1-55.us-west-2.compute.internal",,""ip-10-150-1-55.us-west-2.compute.internal"",,levellevel""::4030""pid"pid"::47594759,","downstairs":upstairs":21}
,,""time"time:{}"":
"2023-09-22T23:14:58.580939642Z""msg,2023-09-22T23:14:58.580939764Z"{hostname"",:"""hostnamemsg""ip-10-150-1-55.us-west-2.compute.internal:"",:"""ip-10-150-1-55.us-west-2.compute.internalpid""28a12560-c07c-41d3-8d35-7e06b0c337aa Active Active Active":,:"4759pid",,""v"":upstairs4759:,"":upstairs1"negotiate packet ExtentVersionsPlease}:
10",{},""name""
msgv:"{":""crucible"msg[0] Starts reconcile loop",:"",""level"::[0]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]""v":,0",v30"":name0":"0crucible,"","name"level:"":,crucible""30,"name"level"::30","crucible"time":",",level"2023-09-22T23:14:58.581025472Z":",,30"timetime""::"""hostname":2023-09-22T23:14:58.58104009Z2023-09-22T23:14:58.581035636Z""",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internal","pid"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,""pid,",:"4759",pidtime":"":4759:"4759upstairs",:2023-09-22T23:14:58.581050948Z",",upstairs":11"}"
upstairs"}hostname"
{::""1msg"}:
"{ip-10-150-1-55.us-west-2.compute.internal{",""pid"":msg":"[1] 85682a66-27fd-403c-bf22-d04b4ff66805 (5d828379-bf79-455f-9270-e82849f9ecba) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorummsg"":,""4759v"Set check for repair",[0]R  dirty: [false, false, false, false, false, false, false, false, false, false]:"0,,""v"name:"0:,""cruciblename"",:,""crucible"level,""""level:"30:v":downstairs":0303,"}name"
:",,""crucible"{timetime""::",""2023-09-22T23:14:58.581131411Z""level"2023-09-22T23:14:58.581126952Z,"","hostname"hostname:"":":msg30"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"::4759pid",:"4759",upstairs"":upstairs1":}1
}
{{","msg":"msg":""[1] Transition from WaitActive to WaitQuorum"time,[1]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]""v,"":v0",:"0name,"":name"":"crucible"":",crucible"",level""level:"[0] 04482c5c-f564-49f3-897c-51c4de58239a (cc12d06e-bd83-45fe-987f-779d8c6339ec) WaitActive WaitActive WaitActive ds_transition to WaitQuorum":3030,2023-09-22T23:14:58.581158345Z""v,"":hostname"0:","name":"ip-10-150-1-55.us-west-2.compute.internal"crucible",",,,""timetime"pid"::"":""47592023-09-22T23:14:58.58119489Z"2023-09-22T23:14:58.581194625Z,"",,hostname"""level"upstairs"hostname:""::"30ip-10-150-1-55.us-west-2.compute.internal":ip-10-150-1-55.us-west-2.compute.internal,"",pid""1pid:"4759:},4759"
,upstairs,"":"upstairs1"}:
1{}{
""timemsg":""{msg""[0] received reconcile message"msg:"":,:""v""[1]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"2023-09-22T23:14:58.581226272Z:,[1] new RM replaced this: None"""v",:"0v,"0":,0,hostname":",""name"name:"":crucible""",crucible""level,""level:"30:name"ip-10-150-1-55.us-west-2.compute.internal":",40"crucible"pid":,"4759level",,:""30time,"":time"upstairs""2023-09-22T23:14:58.581295416Z:"":,12023-09-22T23:14:58.581300187Z""}hostname,"":"hostname"
:ip-10-150-1-55.us-west-2.compute.internal"",","{pidip-10-150-1-55.us-west-2.compute.internal"":,4759""time"pid,"":upstairs4759":,1:"}msg""
upstairs"::{1"2023-09-22T23:14:58.581309444Z""}
msg"[0] Transition from WaitActive to WaitQuorum{,:""msg"":"","[1]R  dirty: [false, false, false, false, false, false, false, false, false, false][1] Starts reconcile loop""hostnamev,,""vv""::00,,""namename""::"""":":cruciblecrucible"",,""0ip-10-150-1-55.us-west-2.compute.internal"levellevel""::3030,","name"pid"::"4759crucible",,""level":upstairs"30:,,""time"time:"":"12023-09-22T23:14:58.581385156Z}2023-09-22T23:14:58.581384879Z"",,""
hostnamehostname""::"","time{ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:"4759:",4759""msg":"upstairs,"":upstairs1"[0] All repairs completed, exit"}:
1,"{}v":":"
msg":02023-09-22T23:14:58.581395829Z"{,""[2]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"name"msg"",:""v"::,""0hostname"[2] 85682a66-27fd-403c-bf22-d04b4ff66805 (5d828379-bf79-455f-9270-e82849f9ecba) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorumcrucible":",,""name"v:,""":level"0crucible,"",name""::level""crucible:"30,"30level"":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759,",upstairs"":time1":,"}"2023-09-22T23:14:58.58147269Z"time,"",
hostname:"":""{time":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.581476461Z""",msg":",pid"":hostname4759"",:"2023-09-22T23:14:58.581474349Z"upstairs":"[0] new RM replaced this: None","ip-10-150-1-55.us-west-2.compute.internal1"},
",hostname{pid":"""v"msg4759::"0",:""upstairs"ip-10-150-1-55.us-west-2.compute.internal":,"1[2]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}"
,,"{pid""vmsg""::0",:""name[2] Transition from WaitActive to WaitQuorum"":,""name"vcrucible"":,0",level":4759:"30name":,""upstairs"crucible":"1crucible",",},time""level:"":
302023-09-22T23:14:58.581561682Z"","level":hostname"40{:"ip-10-150-1-55.us-west-2.compute.internal",,""time"pid:"":"47592023-09-22T23:14:58.58158032Z",,""msghostnameupstairs""::"1"}:,ip-10-150-1-55.us-west-2.compute.internal"""
,"[0] Starts cmd_loop"pid{":"4759msg",:""upstairs"time"::"[2]R  dirty: [false, false, false, false, false, false, false, false, false, false]1"},,
""2023-09-22T23:14:58.581585426Z"{v,"":msg0",:""name":"[2] new RM replaced this: None"v"crucible"",,""vlevelhostname:"0:"""::300ip-10-150-1-55.us-west-2.compute.internal,""name",,"":name,"":pid"":time"":crucible""crucible"4759,2023-09-22T23:14:58.581642092Z,"",level"",:hostname40"":""upstairslevel""::301ip-10-150-1-55.us-west-2.compute.internal,"",time""pid"::"4759}
2023-09-22T23:14:58.581664328Z,"","upstairs":hostname1":"}
ip-10-150-1-55.us-west-2.compute.internal"{,{",pid""msg:"4759:","""msgtimeMax found gen is 1upstairs""",:"1v}"
::"{0",""msgname""::"":"[2] Starts reconcile loopcrucible"",,""vlevel""::030,2023-09-22T23:14:58.58167139Z"[0] Starts reconcile loop","",name"hostname":,""crucibletime"",:""level"::302023-09-22T23:14:58.581717911Z"",""vip-10-150-1-55.us-west-2.compute.internalhostname":"":,","0ip-10-150-1-55.us-west-2.compute.internal""time,"":pid"","pid"name"2023-09-22T23:14:58.581733145Z:"4759::"4759,,"",crucible""upstairshostname""::"1upstairs}"
ip-10-150-1-55.us-west-2.compute.internal":{,,1""msgpid""::""}level
Generation requested: 1 >= found:14759"",,""{vupstairs""::0:"1,}"
30namemsg"{":"":"msg":crucible""[1] received reconcile message","[0] 127.0.0.1:39040 task reports connection:true,""level,":"30v"v"::00,,""namename"":","crucible"time,"":",level:"":crucible"2023-09-22T23:14:58.581813038Z30"",,"time"level"hostname""::,:""time":"ip-10-150-1-55.us-west-2.compute.internal""30,2023-09-22T23:14:58.58183153Z""pid",:"47592023-09-22T23:14:58.581802852Z",hostname"",upstairs:"":"1hostname":ip-10-150-1-55.us-west-2.compute.internal}"
,""pid{",:"4759",msgip-10-150-1-55.us-west-2.compute.internal"":time"upstairs"":""Next flush: 1:,"2023-09-22T23:14:58.58184668Z"pid"1"},:"
,hostname"4759:""{v":"0msg,"":,"ip-10-150-1-55.us-west-2.compute.internal"name",upstairs""85682a66-27fd-403c-bf22-d04b4ff66805 WaitQuorum WaitQuorum WaitQuorum:"":",crucible""pid,v""1:level0",:"30name"":}:"4759
,,crucible""time",:""level":{"302023-09-22T23:14:58.581908678Z"upstairs","":msg"1:"hostname":,""}time
[1] 04482c5c-f564-49f3-897c-51c4de58239a (cc12d06e-bd83-45fe-987f-779d8c6339ec) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum""ip-10-150-1-55.us-west-2.compute.internal:"",{"2023-09-22T23:14:58.581924444Zpid"",:"4759","hostname"upstairs:"":msg,":ip-10-150-1-55.us-west-2.compute.internal1"},"
pid""{":v"4759[1] All repairs completed, exit"",:msg",upstairs"":"1:0"}vAll extents match
"",{","vmsg""::0",":name"[0]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]""name:,""0","cruciblename":""v:"":crucible0",,""namecrucible"level""::"30,crucible",","level"level":"30:,30"leveltime":"":302023-09-22T23:14:58.582005521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",,""timepid""::"4759,"2023-09-22T23:14:58.58201516Z"upstairs":,"1}hostname
",:{"""time"msg":"ip-10-150-1-55.us-west-2.compute.internal:"",2023-09-22T23:14:58.582013962Z"",pidNo downstairs repair required"":"4759,,,time":""""hostname"2023-09-22T23:14:58.582019277Z":vupstairs""":,1:}0
,""{name"hostname:""msg"crucible:"",""level"[0]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]:"30,ip-10-150-1-55.us-west-2.compute.internal"":v":",0",ip-10-150-1-55.us-west-2.compute.internal"pid",","":pid"4759timename",":"upstairs":"":crucible"2023-09-22T23:14:58.582070654Z,""4759,level""hostname:"30::,""1upstairs":}1ip-10-150-1-55.us-west-2.compute.internal"},",pid""

time:":4759"{,{2023-09-22T23:14:58.582099445Z""upstairs"",:"1"}hostname
"msg":{"msg""ip-10-150-1-55.us-west-2.compute.internal"msg,"":pid""::":No initial repair work was required4759","","upstairsv""::01[1] Starts cmd_loop},
""[1] Transition from WaitActive to WaitQuorum",{name"":msg"",crucible:"","""level[0]R  dirty: [false, false, false, false, false, false, false, false, false, false]"":v"30,v:"":v0":,"0name",,"":"nametime"crucible:"":"crucible"",,2023-09-22T23:14:58.582166236Z""level",:"30"hostname"level"::"30ip-10-150-1-55.us-west-2.compute.internal"0,","pid,"time:"4759:",""2023-09-22T23:14:58.582184198Z"upstairs,"":name"hostname1"}:
",":"time"{ip-10-150-1-55.us-west-2.compute.internal":crucible"",msg""pid:"",":2023-09-22T23:14:58.582188982Z"",Set Downstairs and Upstairs active4759"level",,"""hostname:"30:"upstairsv""::10},
"ip-10-150-1-55.us-west-2.compute.internal"{name",""pid"msg:""::"crucible"4759,"[1]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]level"",,",":v30":"0timeupstairs":",1:""},name
2023-09-22T23:14:58.582227423Z""":"time,crucible{""":,""hostname2023-09-22T23:14:58.5822473Z""level,"msg""hostname:"30:":"":ip-10-150-1-55.us-west-2.compute.internal""ip-10-150-1-55.us-west-2.compute.internal",",pid,"""[2] received reconcile message"pid"time:"4759:",,:"2023-09-22T23:14:58.582275426Z""v",upstairs"4759:hostname",0",:"1:}"
ip-10-150-1-55.us-west-2.compute.internal""{,name"upstairs":":""msgpid1""}::4759crucible",""
thread '5c1ab357-2b46-4e37-877a-5089e53cee0e is now active with session: fcec4285-eaa1-46a4-819c-725bdeb5d523upstairs"",:"1v"}:
impacted_blocks::test::extent_from_offset_panics_for_offsets_outside_region' panicked at 'assertion failed: offset.value < ddef.extent_count() as u64 * extent_size', upstairs/src/impacted_blocks.rs:280:5
{,"msg":"[1]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30{"level":30"msg":","[1] new RM replaced this: None"time":","v"2023-09-22T23:14:58.582419009Z":0,,""hostnamename"":":"crucible","ip-10-150-1-55.us-west-2.compute.internal"level,"":,pid"40time"":":47592023-09-22T23:14:58.582432077Z","upstairs,"":hostname":"1}
ip-10-150-1-55.us-west-2.compute.internal",,""time"{:pid"":4759"2023-09-22T23:14:58.582454817Z"msg":",","hostname":upstairs"":[1]R  dirty: [false, false, false, false, false, false, false, false, false, false]"1ip-10-150-1-55.us-west-2.compute.internal,}"
,""v"pid"::4759{0,,""upstairs""name"msg"::1":"}crucible
[2] All repairs completed, exit"",",v"":level"0{,:"30name":""crucible"msg":","level":[1] Starts reconcile loop"30,"v":0The guest has finished waiting for activation
,"name":"crucible",","time":level":"30,"time":"2023-09-22T23:14:58.582528024Z","2023-09-22T23:14:58.582534952Z"hostname":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid":,4759"upstairs",:"1,time":""}
2023-09-22T23:14:58.582550672Z"upstairs":,1"}hostname"
{:"ip-10-150-1-55.us-west-2.compute.internal{""msg,""":pid"msg"::"4759"[2] Starts cmd_loop",","upstairs"[2]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"v",":v":01:,0},
""name"name":"{:crucible"""crucible"msg",,":"level"":level"30:30[2] 04482c5c-f564-49f3-897c-51c4de58239a (cc12d06e-bd83-45fe-987f-779d8c6339ec) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.582631547Z",",hostname":""time":",ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.582630959Z""",,time"""hostname":pid":":"47592023-09-22T23:14:58.582640266Z",,ip-10-150-1-55.us-west-2.compute.internal"",""pid"upstairs"hostname":::"47591},
ip-10-150-1-55.us-west-2.compute.internal"","upstairs"pid"::14759}
,"upstairs":1}
{{"msg":""msg":"[2]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":[2] Transition from WaitActive to WaitQuorum"0,,""name":"v"crucible":,0",level":"30name":"crucible","level":30,","time":"time":"2023-09-22T23:14:58.582722067Z"2023-09-22T23:14:58.582725499Z,"","hostname"hostname"::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",",pid":"4759pid":,"4759upstairs":1,"}upstairs"
:1}
{"msg":"{[2] new RM replaced this: None","v"":msg"0:","name":"crucible","[2]R  dirty: [false, false, false, false, false, false, false, false, false, false]"level":,"40v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.582790952Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid":2023-09-22T23:14:58.582796296Z"4759,","hostname":"upstairs":1}ip-10-150-1-55.us-west-2.compute.internal"
,"pid":4759,{"upstairs":1"}msg
":"[2] Starts reconcile loop"{,"v":0,""namemsg":"":"Max found gen is 1"crucible"0,",",v"level":"0:name,30"":name"":"crucible"crucible",,""levellevel""::3030,"time":"2023-09-22T23:14:58.58285342Z","hostname",,""time"time:"":":"2023-09-22T23:14:58.582857488Z"2023-09-22T23:14:58.582857839Z",","hostname"hostname"::""ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""4759pidpid""::47594759,",,""upstairs":upstairsupstairs""::111}}

{}
{"msg":""msg"Generation requested: 1 >= found:1:"","{v":0,5c1ab357-2b46-4e37-877a-5089e53cee0e Set Active after no repair""name,"":v"""crucible:"0,,""msgnamelevel""::"30"crucible":,""level":30[0] 127.0.0.1:36921 task reports connection:true","v":0,"name":"crucible",,""time":",level":"2023-09-22T23:14:58.582925276Ztime"30",:""hostname":"2023-09-22T23:14:58.58292945Z","hostnameip-10-150-1-55.us-west-2.compute.internal"":,""pid":4759,ip-10-150-1-55.us-west-2.compute.internal"","upstairs"pid:":14759}
,"upstairs":1}{
",msg{":"""Next flush: 1msg"",:""time":"vNotify all downstairs, region set compare is done."":0,","v"name:"0:,""2023-09-22T23:14:58.582942951Znamecrucible"":,"""levelcrucible"":,30","level":hostname30":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":,"1,time"":"time":"}2023-09-22T23:14:58.582977236Z"2023-09-22T23:14:58.582980116Z"
,,""hostname"hostname:"":"{ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759",,msg""":"upstairsupstairs""::11}}

{04482c5c-f564-49f3-897c-51c4de58239a WaitQuorum WaitQuorum WaitQuorum"{","msg"v"msg""::"":0All extents matchSet check for repair"",,""vv""::00,,""name"name:"":"crucible"crucible,"",level""level:"30:,"30name":"crucible","level":30,"time",:""time":"2023-09-22T23:14:58.583040954Z",2023-09-22T23:14:58.583042758Z"",hostname"":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internal"pid",:"4759pid":,4759",upstairs"":1upstairs":}1
},
{"time""{msg":""msg"::No downstairs repair required""","v":[1] 127.0.0.1:47390 task reports connection:true0",,"2023-09-22T23:14:58.583046808Z"name"":v"":crucible",",0","levelname":"30:"hostname"crucible:"","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs",":time1":"},
2023-09-22T23:14:58.583086112Z""time",:""hostname":"{2023-09-22T23:14:58.58309092Z",ip-10-150-1-55.us-west-2.compute.internal"","hostname"pid""::"4759msg":",ip-10-150-1-55.us-west-2.compute.internal"","upstairs"pid:"1:4759},"
upstairs":[0]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]1{}
""msg{":",""v"msgNo initial repair work was required"":,"":v0"5c1ab357-2b46-4e37-877a-5089e53cee0e Active Active Active:"0,,""vname""::0",",namecrucible"":,"""cruciblelevel"",:"30level":name"30:"crucible","level":30,"time":","time":"2023-09-22T23:14:58.583146434Z","2023-09-22T23:14:58.583148681Zhostname"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,",pid"":upstairs4759":1,"},upstairs
":1}
{{"msg":""msg":"Set Downstairs and Upstairs active"Set check for repair",","v"v:":00,,""namename""::""cruciblecrucible"",,""levellevel""::3030",,""time"timetime""::"":"2023-09-22T23:14:58.583182881Z2023-09-22T23:14:58.583182837Z""2023-09-22T23:14:58.583152385Z,,"""hostnamehostname""::"","hostname":"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""ip-10-150-1-55.us-west-2.compute.internal"::47594759,,,"""upstairsupstairs""::11}}

{pid":{"4759msg"":msg"",":"upstairs"[2] 127.0.0.1:35000 task reports connection:true":,"1v85682a66-27fd-403c-bf22-d04b4ff66805 is now active with session: 5d828379-bf79-455f-9270-e82849f9ecba""},:"0v,"":name0",:""name"crucible:""
crucible,"","level"level:":3030{"msg":"[0]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":",,""timetime""::""crucible",2023-09-22T23:14:58.583248165Z"2023-09-22T23:14:58.583248005Z,"""hostname,"":"hostname"level":ip-10-150-1-55.us-west-2.compute.internal:30"","pid":4759ip-10-150-1-55.us-west-2.compute.internal",,""pidupstairs""::47591},
"upstairs":1{}
"msg":"{"msg":"85682a66-27fd-403c-bf22-d04b4ff66805 Set Active after no repair",",v5c1ab357-2b46-4e37-877a-5089e53cee0e Active Active Active"":,0",v"":name0",:""name"crucible:"","crucible"level,"""level:"30:time":"302023-09-22T23:14:58.583272183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,",time,"":time""":2023-09-22T23:14:58.58330024Z""upstairs,2023-09-22T23:14:58.583302498Z"","hostnamehostname"":":"":1ip-10-150-1-55.us-west-2.compute.internal"},ip-10-150-1-55.us-west-2.compute.internal""pid,"":
pid4759":4759,","upstairs":upstairs"1:{}1
}"
{msg":"{"msg":""msg":"[0]R  dirty: [false, false, false, false, false, false, false, false, false, false]Notify all downstairs, region set compare is done.Set check for repair""",,"",vv""::00,,""name""name:"":crucible""vcrucible,"","levellevel""::3030":0,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:14:58.583362221Z2023-09-22T23:14:58.58336178Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid"",::47594759",,""time"upstairsupstairs""::11:"}}

{2023-09-22T23:14:58.583367692Z""{msg","":"msg"hostnameSet check for repair:"",":"[0] received reconcile messagev"",":v0",:"0name,"":name"":"cruciblecrucible"",,"""levellevel""::3030ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,,""timetime""::""{2023-09-22T23:14:58.583416969Z2023-09-22T23:14:58.583416863Z"",,""hostnamehostname"""msg::""":"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""[1]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"pidpid""::47594759,",,v":""0upstairsupstairs""::11,"}name"}

{:"crucible{"""msg"msg:"":","level":[1] 127.0.0.1:62630 task reports connection:true[0] All repairs completed, exit"30,""v,"":v"0:,0","name":name"":"crucible"crucible,"","level"level:":3030,"time":",,""timetime""::""2023-09-22T23:14:58.583472441Z"2023-09-22T23:14:58.583478235Z2023-09-22T23:14:58.583478276Z"",,,"""hostnamehostname""::""hostname":"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",ip-10-150-1-55.us-west-2.compute.internal"",pid""pid:"4759:,,4759""upstairs,pid""":upstairs1":}:
14759}
{,"{"upstairs"msgmsg""::""":[0] Starts cmd_loop1"85682a66-27fd-403c-bf22-d04b4ff66805 Active Active Active",,""vv""::00,,""name"name}"::""
cruciblecrucible"",,""levellevel""::3030{"msg":"[1]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible",,""time":"time":,2023-09-22T23:14:58.58354374Z""","2023-09-22T23:14:58.583543687Zhostname"",":"hostname"level":ip-10-150-1-55.us-west-2.compute.internal:"",30"pid"ip-10-150-1-55.us-west-2.compute.internal:"4759,"pid,"":4759upstairs":,"1upstairs}"
:1}
{{"msg":""msg":"[1] received reconcile message"Set check for repair,"",v"":v0",:"0name,"":name"":crucible"",,crucible""",level""level:"30:time":30"2023-09-22T23:14:58.583569324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",",pid",""time"time:"":":2023-09-22T23:14:58.583595952Z"2023-09-22T23:14:58.583598205Z",","hostname"hostname:"":4759"ip-10-150-1-55.us-west-2.compute.internal",",pid"ip-10-150-1-55.us-west-2.compute.internal:"4759,","upstairs":pid""1upstairs:":47591}},
"
{upstairs":1}"
msg"{{:""msg":[1] All repairs completed, exit""","msg[2] 127.0.0.1:41929 task reports connection:truev"",:"0v",:"0name,"":":"name"":"crucible"crucible,"","level":level30":30[1]R  dirty: [false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:14:58.58365467Z2023-09-22T23:14:58.583655553Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759,,""upstairsupstairs""::11,}}

{"time""{msg":""msg":[1] Starts cmd_loop:""","v":02023-09-22T23:14:58.583662509Z85682a66-27fd-403c-bf22-d04b4ff66805 Active Active Active,"","name""v:"","crucible:"0,,""level"name:"30hostname":":"crucible","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":,"1time":}"
2023-09-22T23:14:58.583702414Z,"",time"":"hostname":"{2023-09-22T23:14:58.583707413Z"ip-10-150-1-55.us-west-2.compute.internal,"","hostnamepid"":":"4759msg",:"ip-10-150-1-55.us-west-2.compute.internal"","upstairs"pid:":14759}[2]R  flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"
,","upstairs{":v"1msg"}:
"":0{[2] received reconcile message"","msg,"":v"":name"Set check for repair0",:",""namev""::"0crucible",crucible"",,name""level:"":"crucible30"level":,30"level":30,"time":"2023-09-22T23:14:58.583765612Z",","hostname"time:"":"2023-09-22T23:14:58.58376961Z"ip-10-150-1-55.us-west-2.compute.internal",,""pidhostname""::"4759,",ip-10-150-1-55.us-west-2.compute.internal"","upstairs"pid:"1:}4759
,"time"{upstairs":"":msg1":"}
[2] All repairs completed, exit"2023-09-22T23:14:58.583768504Z",{"v"":msg0",:""name",:[0] received reconcile message"""crucible,"",v"hostname"level""::030,":"name":"crucible",ip-10-150-1-55.us-west-2.compute.internal"","level":pid":304759,"time,":""upstairs":2023-09-22T23:14:58.583816334Z"1,"}hostname",:""time":"
ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.58382543Z"",","pid":hostname4759":","{upstairs"ip-10-150-1-55.us-west-2.compute.internal:"1,"}
pid"":{4759msg":,""msg"upstairs:"":"[2] Starts cmd_loop1"}
,[2]R  generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"{"v"":,0msg,"":name"":""crucible[0] All repairs completed, exit"",",v"level""v:"30::00,,""namename":":""cruciblecrucible,"",time""level:"":"302023-09-22T23:14:58.583872048Z",,""hostname"level"::"30ip-10-150-1-55.us-west-2.compute.internal,"","time"pid:"":4759,"2023-09-22T23:14:58.583886302Z"upstairs",:"1hostname}"
:","time"ip-10-150-1-55.us-west-2.compute.internal":","pid":47592023-09-22T23:14:58.583892841Z",","upstairs":hostname"1:}
"{ip-10-150-1-55.us-west-2.compute.internal",""pid":msg"4759:","[0] Starts cmd_loop"upstairs",:"1v":}0
,"name":"crucible","level":{30"msg":"[2]R  dirty: [false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level,""time":":302023-09-22T23:14:58.583942631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,"time":"{2023-09-22T23:14:58.583958889Z",""hostnamemsg":"":"[1] received reconcile message","vip-10-150-1-55.us-west-2.compute.internal"",":pid":04759,"name",:""crucible"upstairs",:"1level":}30
{"msg":"Max found gen is 1","v":0,"name":","crucible"time":","level":2023-09-22T23:14:58.583994618Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,"time{":"2023-09-22T23:14:58.584014977Z"",msg":""hostname":"[1] All repairs completed, exit","v":ip-10-150-1-55.us-west-2.compute.internal"0,,""name":"pid"crucible":,"4759level":,30"upstairs":1}
{"msg":"Generation requested: 1 >= found:1",","v"time"::0","name":2023-09-22T23:14:58.58405039Z"",crucible"","hostname"level":":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{,"time":""msg":"2023-09-22T23:14:58.584078111Z"[1] Starts cmd_loop",","hostname":"v":0,"ip-10-150-1-55.us-west-2.compute.internalname"":","crucible"pid":,4759"level",:"30upstairs":1}
{"msg":"Next flush: 1","v":0,"name":","crucible"time",:""level":302023-09-22T23:14:58.58411135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
,"time":"{2023-09-22T23:14:58.584132467Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal"[2] received reconcile message",","pidv":"0:,4759"name,":""crucible"upstairs",":level":130,"time":"2023-09-22T23:14:58.584169606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal}"
,"pid":4759,"{upstairs":1}
"msg":"All extents match","{v":0,"name":""msg":"crucible","[2] All repairs completed, exitlevel":"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584220147Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",","pid":time4759":","upstairs2023-09-22T23:14:58.584226423Z"":,"1hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759{,"upstairs":1"}msg
":"No downstairs repair required","{v":0,"name":""cruciblemsg":"",[2] Starts cmd_loop""level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584280644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",,""pid"time:":"4759,2023-09-22T23:14:58.584285709Z"",upstairs":"1hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759{,"upstairs":1"}msg
":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584337281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584378716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"04482c5c-f564-49f3-897c-51c4de58239a is now active with session: cc12d06e-bd83-45fe-987f-779d8c6339ec","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584420237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"04482c5c-f564-49f3-897c-51c4de58239a Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584463465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584503717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584549392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] 127.0.0.1:37306 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584592802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"04482c5c-f564-49f3-897c-51c4de58239a Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584632875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584671702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] 127.0.0.1:55683 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584711618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"04482c5c-f564-49f3-897c-51c4de58239a Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584752377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584790172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584831342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584869806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584908696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584947711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.584987261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.58502715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.585065257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.585105667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.585144173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] downstairs disconnected","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:58.585206215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 04482c5c-f564-49f3-897c-51c4de58239a Gone missing, transition from Active to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.585250497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 04482c5c-f564-49f3-897c-51c4de58239a connection to 127.0.0.1:36921 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.585292318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:58.585333917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 127.0.0.1:36921 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.585375426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"04482c5c-f564-49f3-897c-51c4de58239a Offline Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.585421955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 127.0.0.1:36921 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.585462782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
test impacted_blocks::test::extent_from_offset_can_recreate_iblocks ... ok
test impacted_blocks::test::iblocks_conflicts_is_commutative ... ok
thread 'impacted_blocks::test::extent_from_offset_panics_when_num_blocks_outside_region' panicked at 'assertion failed: offset.value as u128 + num_blocks.value as u128 <=\n    ddef.extent_count() as u128 * extent_size as u128', upstairs/src/impacted_blocks.rs:284:5
test impacted_blocks::test::extent_from_offset_panics_for_offsets_outside_region ... ok
test impacted_blocks::test::iblocks_extents_returns_correct_extents ... ok
thread 'impacted_blocks::test::extent_from_offset_panics_when_num_blocks_outside_region' panicked at 'assertion failed: offset.value as u128 + num_blocks.value as u128 <=\n    ddef.extent_count() as u128 * extent_size as u128', upstairs/src/impacted_blocks.rs:284:5
test impacted_blocks::test::extent_from_offset_panics_when_num_blocks_outside_region ... ok
test impacted_blocks::test::iblocks_from_offset_is_empty_for_zero_blocks ... ok
thread 'impacted_blocks::test::iblocks_from_offset_with_zero_extent_size_panics' panicked at 'assertion failed: extent_size > 0', upstairs/src/impacted_blocks.rs:146:9
test impacted_blocks::test::iblocks_from_offset_with_zero_extent_size_panics ... ok
test impacted_blocks::test::intersection_is_associative ... ok
test impacted_blocks::test::intersection_is_commutative ... ok
test impacted_blocks::test::intersection_with_empty_is_empty ... ok
test impacted_blocks::test::iblocks_new_panics_for_flipped_polarity ... ok
test impacted_blocks::test::intersection_produces_less_than_or_equal_block_count ... ok
test impacted_blocks::test::nothing_contains_empty ... ok
test impacted_blocks::test::test_extent_from_offset ... ok
test impacted_blocks::test::test_extent_from_offset_single_block_only ... ok
test impacted_blocks::test::test_extent_to_impacted_blocks ... ok
test impacted_blocks::test::test_impacted_blocks_from_offset ... ok
test impacted_blocks::test::test_large_extent_to_impacted_blocks ... ok
test impacted_blocks::test::test_new_range_panics_when_last_block_before_first - should panic ... ok
test impacted_blocks::test::test_new_range_panics_when_last_extent_before_first - should panic ... ok
test impacted_blocks::test::overlapping_impacted_blocks_should_conflict ... ok
test impacted_blocks::test::union_is_associative ... ok
test impacted_blocks::test::union_is_commutative ... ok
test impacted_blocks::test::subregions_are_contained ... ok
{"msg":"Crucible stats registered with UUID: 641ab82c-3ee3-4691-881a-11c5f52cf819","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.670910897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 641ab82c-3ee3-4691-881a-11c5f52cf819 has session id: a2e2ade3-c458-46cf-a429-7df7c085dacd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.670985693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"641ab82c-3ee3-4691-881a-11c5f52cf819 is now active with session: 46312f17-e793-483e-9dca-95375fb17903","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671031039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67107272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671114262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671162956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671202375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671244138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67128166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671324848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671361365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671403779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671442148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67148235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671520632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671561038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671600561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67164048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671679429Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671720833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67175806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671799501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671843658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671886176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.671924305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67196652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.6720046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.672045213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.672084023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 641ab82c-3ee3-4691-881a-11c5f52cf819 (46312f17-e793-483e-9dca-95375fb17903) Faulted LiveRepair Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.672124366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.672164291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.672199926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Upstairs already in repair, trying again later","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:58.672238305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test impacted_blocks::test::union_produces_greater_than_or_equal_block_count ... ok
test live_repair::repair_test::test_check_for_repair_already_repair ... ok
{"msg":"Crucible stats registered with UUID: 8228decb-4d72-4851-9eb0-d687894e47e9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.673029552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 8228decb-4d72-4851-9eb0-d687894e47e9 has session id: b5e0c294-607f-4c30-a2e2-eba071706399","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.673083887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"8228decb-4d72-4851-9eb0-d687894e47e9 is now active with session: 4dbfcf6c-f885-44e1-a38c-eeeb7a138dad","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67312453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.673163474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30{"msg":"Crucible stats registered with UUID: 120e0f16-155b-4b25-9c8c-e48b4995715e","v":0,"name":"crucible",,""time"level:"":302023-09-22T23:14:58.67319914Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible,"","time"level:"":302023-09-22T23:14:58.673214482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.673240008Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
Crucible 120e0f16-155b-4b25-9c8c-e48b4995715e has session id: 320a8300-6268-4943-be65-a9ba6f11e562","v"{:0,""name"msg:"":"crucible","level":[0] Transition from WaitActive to WaitQuorum30","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:14:58.673272725Ztime"":","hostname":2023-09-22T23:14:58.6732774Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759ip-10-150-1-55.us-west-2.compute.internal","}pid
":4759}
{{"msg":""msg":"120e0f16-155b-4b25-9c8c-e48b4995715e is now active with session: b41d05a7-cb34-4f00-b2d8-90477e36239b","v":[0] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) WaitQuorum New New ds_transition to Active0",","namev""::"0,crucible""name,"":"level":crucible30","level":30,",time"":time"":"2023-09-22T23:14:58.673312065Z"2023-09-22T23:14:58.673313586Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:":47594759}}

{"{msg":""msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","[0] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) New New New ds_transition to WaitActive"level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.673344262Z",","hostnametime""::""2023-09-22T23:14:58.673348338Z"ip-10-150-1-55.us-west-2.compute.internal",,""pidhostname""::"4759}
ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"msg":"{"msg":"[0] Transition from New to WaitActive",[1] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) Active New New ds_transition to WaitActive""v",:"0v",:"0,name"":name"":"crucible"crucible,"","level":level"30:30,,""timetime""::""2023-09-22T23:14:58.673380608Z2023-09-22T23:14:58.673380928Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{"{msg":""msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","[0] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) WaitActive New New ds_transition to WaitQuorumlevel"":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.673413571Z",",hostname"":"time":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.673417822Z"",","pid":hostname4759":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"msg":"{"msg":"[0] Transition from WaitActive to WaitQuorum","[1] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) Active WaitActive New ds_transition to WaitQuorumv"":,"0v",:"0name,"":"name":crucible"",crucible"",level"":level30":30,,""timetime""::""2023-09-22T23:14:58.67344989Z2023-09-22T23:14:58.673450321Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{"{msg":""msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","[0] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) WaitQuorum New New ds_transition to Active"level":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.673482289Z",","hostname":time"":"2023-09-22T23:14:58.673486568Zip-10-150-1-55.us-west-2.compute.internal"",,""pid":hostname"4759:"}
ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"msg":"{"msg":"[0] Transition from WaitQuorum to Active","[1] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) Active WaitQuorum New ds_transition to Activev"":,0",v"":name0":,""namecrucible"":","cruciblelevel"":,"30level":30,,""timetime""::""2023-09-22T23:14:58.673519054Z2023-09-22T23:14:58.673518251Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{{"msg"":"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible[1] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active New New ds_transition to WaitActive"",","levelv""::300,"name":"crucible","level":30,"time":","time"2023-09-22T23:14:58.673557296Z:"","hostname"2023-09-22T23:14:58.673560704Z:"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid}"
:4759}
{{"msg":""msg":"[1] Transition from New to WaitActive","v":0,"name"[2] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) Active Active New ds_transition to WaitActive:"","crucible"v",:"0,level"":name30":"crucible","level":30,"time":","time"2023-09-22T23:14:58.673592314Z:"","hostname"2023-09-22T23:14:58.673594847Z:"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal4759","}
pid":4759}{
"msg":"{"msg":"[2] Transition from New to WaitActive","v":[1] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active WaitActive New ds_transition to WaitQuorum0",","namev""::"0,crucible"",name"":"level":crucible"30,"level":30,",time"":time"":"2023-09-22T23:14:58.673626653Z"2023-09-22T23:14:58.673627981Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:":47594759}}

{{"msg":""msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name[2] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) Active Active WaitActive ds_transition to WaitQuorum"":","cruciblev"":,"0,level"":name30":"crucible","level":30,"time":","time2023-09-22T23:14:58.673659898Z"":","hostname":2023-09-22T23:14:58.673662427Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759ip-10-150-1-55.us-west-2.compute.internal",}"
pid":4759}{
"msg":"{"msg":"[2] Transition from WaitActive to WaitQuorum","[1] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active WaitQuorum New ds_transition to Active"v":,"0v,"":0name",:""name"crucible:"","crucible"level",:"30level":30,,""timetime""::""2023-09-22T23:14:58.673694403Z2023-09-22T23:14:58.673695167Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:"4759:4759}
}
{{"msg":""msg":"[1] Transition from WaitQuorum to Active","v":0,"name[2] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) Active Active WaitQuorum ds_transition to Active"":","cruciblev"":,0","level"name:"30:"crucible","level":30,"time":","time":2023-09-22T23:14:58.673729033Z"","hostname2023-09-22T23:14:58.673731375Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid}"
:4759}
{{"msg":""msg":"[2] Transition from WaitQuorum to Active","v":0,"[2] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active Active New ds_transition to WaitActivename"":","vcrucible"":,0","level"name:":30"crucible","level":30,"time":","time":2023-09-22T23:14:58.673763416Z"","hostname2023-09-22T23:14:58.673765649Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"}pid
":4759}
{{"msg":""msg":"[2] Transition from New to WaitActive","v":0,"name[1] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) Active Active Active ds_transition to Faulted"":","cruciblev"":,"0,level"":name30":"crucible","level":30,"time":","time":2023-09-22T23:14:58.673796954Z"","hostname2023-09-22T23:14:58.673799431Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal4759","}
pid":4759}{
"msg":"{"msg":"[1] Transition from Active to Faulted","v":[2] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active Active WaitActive ds_transition to WaitQuorum0",","namev""::"0,crucible""name,"":"level":crucible"30,"level":30,,""timetime""::""2023-09-22T23:14:58.673831395Z2023-09-22T23:14:58.673832747Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{{"msg"":msg"":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"[1] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) Active Faulted Active ds_transition to LiveRepairReady"crucible",,""v"level:":030,"name":"crucible","level":30,"time":","2023-09-22T23:14:58.673869227Z"time":,""hostname":"2023-09-22T23:14:58.673872463Z","hostname"ip-10-150-1-55.us-west-2.compute.internal:"","pid":4759ip-10-150-1-55.us-west-2.compute.internal"},
"pid":4759{}
"msg":"{"msg":"[1] Transition from Faulted to LiveRepairReady",[2] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active Active WaitQuorum ds_transition to Active""v",:"0v",:"0,name"":name"":"crucible"crucible,"","level":level"30:30,,""timetime""::""2023-09-22T23:14:58.673905295Z2023-09-22T23:14:58.673905561Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid:":47594759}}

{{"msg":""msg":"Checking if live repair is needed","v[2] Transition from WaitQuorum to Active"":0,","v":name"0:","cruciblename"":,""cruciblelevel"":,"30level":30,,""timetime""::""2023-09-22T23:14:58.673938603Z2023-09-22T23:14:58.673939444Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{{"msg":""msg":"[1] 8228decb-4d72-4851-9eb0-d687894e47e9 (4dbfcf6c-f885-44e1-a38c-eeeb7a138dad) Active LiveRepairReady Active ds_transition to LiveRepair"[1] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active Active Active ds_transition to Faulted",","v"v:":00,,""namename""::""crucible"crucible",","level"level:"30:30,,""timetime""::""2023-09-22T23:14:58.673976414Z2023-09-22T23:14:58.673976138Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:":47594759}}

{{"msgtest impacted_blocks::test::union_with_empty_is_identity ... ""msg:"":ok[1] Transition from Active to Faulted""
[1] Transition from LiveRepairReady to LiveRepair,""v":,0",v""name"::0","crucible"name",:""level"crucible:"30,"level":30,","time"time:"":"2023-09-22T23:14:58.674027398Z"2023-09-22T23:14:58.674028697Z",","hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:":47594759}}

{"msg":"[1] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674062377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674097184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active LiveRepairReady Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674136287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674173163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active LiveRepairReady Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67421467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674253884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"test live_repair::repair_test::test_check_for_repair_do_repair ... 2023-09-22T23:14:58.674289781Z"ok,"
hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active LiveRepairReady LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674337919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674359087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 120e0f16-155b-4b25-9c8c-e48b4995715e (b41d05a7-cb34-4f00-b2d8-90477e36239b) Active LiveRepair LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674388117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674407525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: a1d81752-deda-403e-8082-906a34f8182a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674593437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internaltest live_repair::repair_test::test_check_for_repair_do_two_repair ... "ok,"
pid":4759}
{"msg":"Crucible a1d81752-deda-403e-8082-906a34f8182a has session id: 88d926f8-def9-4076-bba6-faf2ecc35512","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674660486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674694514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"a1d81752-deda-403e-8082-906a34f8182a is now active with session: 7ab251f4-fbab-4f08-b351-a01533f527e9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.674728571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] a1d81752-deda-403e-8082-906a34f8182a (7ab251f4-fbab-4f08-b351-a01533f527e9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67476288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30{"msg":","time":"2023-09-22T23:14:58.674793383ZCrucible stats registered with UUID: d90599f9-c15d-4f80-b9c1-b848e2e8d476"",",hostname"":v"":0,"ip-10-150-1-55.us-west-2.compute.internal"name",":"pid":crucible"4759,"}level"
:30{"msg":"[0] a1d81752-deda-403e-8082-906a34f8182a (7ab251f4-fbab-4f08-b351-a01533f527e9) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,",time":""time":"2023-09-22T23:14:58.674838418Z","2023-09-22T23:14:58.674825568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"hostname":"Crucible d90599f9-c15d-4f80-b9c1-b848e2e8d476 has session id: 1eae53c6-8d73-4b5f-801d-5e6e237195b1","v":ip-10-150-1-55.us-west-2.compute.internal0",",name":"crucible","level":30"pid":4759}
{"msg":",[0] Transition from WaitActive to WaitQuorum""time",:""v":0,2023-09-22T23:14:58.674932173Z""name":","crucible"hostname",:""level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":""msg":2023-09-22T23:14:58.674963935Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal",d90599f9-c15d-4f80-b9c1-b848e2e8d476 is now active with session: 4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3""pid":,4759"v"}:
0,"name":"{crucible",""levelmsg":"":30[0] a1d81752-deda-403e-8082-906a34f8182a (7ab251f4-fbab-4f08-b351-a01533f527e9) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:14:58.675002309Z"time":","hostname":"2023-09-22T23:14:58.675011854Z","hostname"ip-10-150-1-55.us-west-2.compute.internal":,""pid":4759ip-10-150-1-55.us-west-2.compute.internal",}"
{pid""msg"::4759"}
[0] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) New New New ds_transition to WaitActive","v":{0,"name":""cruciblemsg"":,""level":30[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:14:58.675070066Ztime":"","hostname"2023-09-22T23:14:58.675076395Z":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid":}4759
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30}
,"time":"2023-09-22T23:14:58.675164625Z"{,"hostname":""ip-10-150-1-55.us-west-2.compute.internal"msg",:""pid":4759}
{"msg":"[1] a1d81752-deda-403e-8082-906a34f8182a (7ab251f4-fbab-4f08-b351-a01533f527e9) Active New New ds_transition to WaitActive","v":[0] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) WaitActive New New ds_transition to WaitQuorum0",",name"":"v"crucible:0,"name":"crucible","level":30","level":30,"time":"2023-09-22T23:14:58.675233101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",",{"pid"time"::4759"msg":"Crucible stats registered with UUID: c7bd94b4-dec0-4286-b77a-1e856258195c"}
"{2023-09-22T23:14:58.675240979Z",""hostname"msg:""ip-10-150-1-55.us-west-2.compute.internal",":pid"":4759[0] Transition from WaitActive to WaitQuorum}"
,{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30"v":,"0time":,""name":"crucible"2023-09-22T23:14:58.675314474Z",","level":hostname"30,"v":0,"name":"crucible","level":30:"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,}"
time":"2023-09-22T23:14:58.675345702Z","{hostname":""ip-10-150-1-55.us-west-2.compute.internalmsg"":","pid":4759}
{[1] a1d81752-deda-403e-8082-906a34f8182a (7ab251f4-fbab-4f08-b351-a01533f527e9) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":""2023-09-22T23:14:58.675408485Z"msg",:",[0] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) WaitQuorum New New ds_transition to Active",""v":hostname0",:""name":"crucibleip-10-150-1-55.us-west-2.compute.internal"""time",":pid"":,"47592023-09-22T23:14:58.675352816Z"level"}:
,"30hostname":"ip-10-150-1-55.us-west-2.compute.internal{","pid":4759"msg":"}
[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible"{,,""timelevel""::30"2023-09-22T23:14:58.675468963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}"
{,"time":"2023-09-22T23:14:58.675493179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] a1d81752-deda-403e-8082-906a34f8182a (7ab251f4-fbab-4f08-b351-a01533f527e9) Active WaitQuorum New ds_transition to Active","v":0","msgname""::"[0] Transition from WaitQuorum to Active"msg,""v"::"0,"name":""crucible",crucible""Crucible c7bd94b4-dec0-4286-b77a-1e856258195c has session id: ef6e50e6-6ec1-49ef-aac4-a434e5e75199",level","v":0,"name":":crucible"30,"level":30"level":30,"time":"2023-09-22T23:14:58.675579141Z",","time"hostname":","time":"2023-09-22T23:14:58.675588193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal4759","pid":}4759
{:"2023-09-22T23:14:58.675581361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":""[0] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) New New New ds_transition to WaitActive"msg":,""v":0,"name":"crucible","level":[1] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) Active New New ds_transition to WaitActive"30,"v":0,"name":"crucible","level":30}
,"time":"2023-09-22T23:14:58.675669132Z",{"hostname":",""ip-10-150-1-55.us-west-2.compute.internal"msg,time"""pid::""2023-09-22T23:14:58.675679675Z"[1] Transition from WaitQuorum to Active",","hostname"v":""ip-10-150-1-55.us-west-2.compute.internal":,"4759pid":4759}
}
:0,"name":"crucible","level{":30"{msg":"[1] Transition from New to WaitActive","v":"0,"msg"name":"crucible","level",:"30time":"2023-09-22T23:14:58.675746657Z":,""hostname":"[0] Transition from New to WaitActive"ip-10-150-1-55.us-west-2.compute.internal",",pid"":v"4759:0},
,""name":"time{"crucible":",""level":2023-09-22T23:14:58.675766432Zmsg30","time":"2023-09-22T23:14:58.675800799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","hostname":"[0] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) WaitActive New New ds_transition to WaitQuorumip-10-150-1-55.us-west-2.compute.internal"","pid,""v"::04759,"name":"}crucible"
{,"level":"30msg":"[1] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) Active WaitActive New ds_transition to WaitQuorum,"time":"2023-09-22T23:14:58.675863639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30","time","v:"":02023-09-22T23:14:58.67589105Z,""","name:"":[2] a1d81752-deda-403e-8082-906a34f8182a (7ab251f4-fbab-4f08-b351-a01533f527e9) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30"crucible",,hostname":"ip-10-150-1-55.us-west-2.compute.internal"","level"pid"::475930"}
time":"{2023-09-22T23:14:58.675924754Z"",msg":""hostname":"[0] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) WaitQuorum New New ds_transition to Activeip-10-150-1-55.us-west-2.compute.internal"",","v":pid"0:,,""nametime":"2023-09-22T23:14:58.6759729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{""msg":":"crucible"[1] Transition from WaitActive to WaitQuorum",",level""v"::300,"name":"crucible","level":475930}
,"time":"{2023-09-22T23:14:58.676026942Z","hostname":""ip-10-150-1-55.us-west-2.compute.internalmsg"":,""pid,":"4759[2] Transition from New to WaitActive"}time
"{"msg":":"[0] Transition from WaitQuorum to Active","v":02023-09-22T23:14:58.676031673Z,""name":",,""v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.676073211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
hostname":"{ip-10-150-1-55.us-west-2.compute.internal",""pidmsg""::"4759}
crucible","level":30,"time":"2023-09-22T23:14:58.67612504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":"[2] a1d81752-deda-403e-8082-906a34f8182a (7ab251f4-fbab-4f08-b351-a01533f527e9) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30[1] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":""2023-09-22T23:14:58.6761637Z"msg":","hostname":"[1] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) Active New New ds_transition to WaitActive"ip-10-150-1-55.us-west-2.compute.internal",","pid"v"::04759,"name":"}crucible
",,""level":time"30:"2023-09-22T23:14:58.67619352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30{"msg":","time":"[2] Transition from WaitActive to WaitQuorum"2023-09-22T23:14:58.676242401Z",","vhostname":"":0,ip-10-150-1-55.us-west-2.compute.internal"",,""pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.676281293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"time":"2023-09-22T23:14:58.676250459Z","hostname":"[1] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) Active WaitActive New ds_transition to WaitQuorum","ip-10-150-1-55.us-west-2.compute.internalv":"0,",name":""crucible"pid,""level":30,"time":"2023-09-22T23:14:58.676315746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30:4759}
name":"crucible","level":30,"time":"2023-09-22T23:14:58.676355253Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759"}msg":","time":"2023-09-22T23:14:58.676336673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30[2] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.676411276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":","crucible"time"
{"msg":"[2] a1d81752-deda-403e-8082-906a34f8182a (7ab251f4-fbab-4f08-b351-a01533f527e9) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible:"","level":2023-09-22T23:14:58.676425028Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.676480391Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal",,"[2] Transition from New to WaitActivepid"":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":",2023-09-22T23:14:58.676525486Z""v":,"0hostname",:""name":""levelip-10-150-1-55.us-west-2.compute.internal"crucible","","level":30pid":4759}
,"time":"2023-09-22T23:14:58.676554972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""msg":"Checking if live repair is needed","v":0[2] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) Active Active WaitActive ds_transition to WaitQuorum",",name":""v":crucible"0,",:"levelname":"30:,"time":"2023-09-22T23:14:58.676613278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":""crucible","No Live Repair required at this timelevel"",:"30v":0,30"name":"crucible","level":30,,""timetime""::",""2023-09-22T23:14:58.67665563Z"2023-09-22T23:14:58.676653004Z"time,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,":time"":"2023-09-22T23:14:58.676650502Z2023-09-22T23:14:58.676696792Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.676726708Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal",hostname""pid"::"4759}
ip-10-150-1-55.us-west-2.compute.internal{"msg":"","pid":4759[2] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) Active Active WaitActive ds_transition to WaitQuorum","}v
{":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.676774135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible"","level"msg":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.676837298Z",":hostname":30"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.676863179Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"msg":"{"msg":"[2] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) Active Active WaitQuorum ds_transition to Active","v":0,"name":"[2] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) Active Active WaitQuorum ds_transition to Active"crucible",",v":"0level":,"30name":"crucible","level":30,"time":"2023-09-22T23:14:58.676903041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",,""pid":4759time":"}
2023-09-22T23:14:58.676900226Z"{,""hostname":msg":""[2] Transition from WaitQuorum to Active","v":ip-10-150-1-55.us-west-2.compute.internal0",","pid":4759}
{name":"crucible",""level":msg"30:"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.676985358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"c7bd94b4-dec0-4286-b77a-1e856258195c is now active with session: 148894f2-0fa3-4b3d-a2ba-6c769977de7b","v":0,"name":,""crucible",time""level":30:"2023-09-22T23:14:58.676994516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",","pid"time":":47592023-09-22T23:14:58.677017227Z","hostname":"}
{"msg":ip-10-150-1-55.us-west-2.compute.internal"","pid":4759}
[1] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) Active Active Active ds_transition to Faulted","v":{0,""namemsg":"":"crucible","level":30[1] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"test live_repair::repair_test::test_check_for_repair_normal ... time":,"time":"2023-09-22T23:14:58.677106484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30"2023-09-22T23:14:58.677098906Z",,""time":"hostname":"2023-09-22T23:14:58.677143652Z","ip-10-150-1-55.us-west-2.compute.internal"hostname",:""pid":4759ip-10-150-1-55.us-west-2.compute.internal",}"
{pid":4759}
{"msg":"[1] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible",",level":30,"time":"2023-09-22T23:14:58.677253789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759"time":"}
ok{
"msg":"[1] d90599f9-c15d-4f80-b9c1-b848e2e8d476 (4bd7dc1d-d4df-4f2e-8daa-2d91d52e68c3) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","2023-09-22T23:14:58.677237011Zlevel"":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:14:58.67734569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time"":"msg":"2023-09-22T23:14:58.677365797Z","hostname":[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] c7bd94b4-dec0-4286-b77a-1e856258195c (148894f2-0fa3-4b3d-a2ba-6c769977de7b) Active LiveRepairReady Active ds_transition to LiveRepair",",v":0","timename":""crucible":,""level":302023-09-22T23:14:58.677432222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":""msg2023-09-22T23:14:58.677458485Z"":","hostname":"Checking if live repair is needed"ip-10-150-1-55.us-west-2.compute.internal","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.677523762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg",:""pid":4759}
Upstairs repair task running, trying again later","v":{0,""name":"crucible","level":40msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.677611738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.677627058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 1cff5a3f-832b-41e6-a1da-ab29a77accd1","v":0,"name":"crucible"test live_repair::repair_test::test_live_repair_deps_after_no_overlap ... ,"oklevel
":30,"time":"2023-09-22T23:14:58.677861296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_check_for_repair_task_running ... ok{
"msg":"Crucible 1cff5a3f-832b-41e6-a1da-ab29a77accd1 has session id: d0cad690-982c-473a-b981-845555a75a75","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.677912023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.677952722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.677991106Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67803907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678082403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678123297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678159804Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678198489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678232161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67827289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678304865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678342774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678380746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"msg":"{"msg":"Crucible stats registered with UUID: b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5","v":0,"name":"[2] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) Active Active New ds_transition to WaitActive"crucible",","v":level"0:,"30name":"crucible","level":30,"time":"2023-09-22T23:14:58.678427995Z,"","time":hostname":""2023-09-22T23:14:58.678425306Z"ip-10-150-1-55.us-west-2.compute.internal",,""pid":hostname"4759:"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759{}
"msg":"[2] Transition from New to WaitActive"{,"v":0,""msg"name":":"crucible","level":30Crucible b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 has session id: 56decb8e-bf1d-4f14-a929-82d627317444","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678485856Z",","hostname"time"::""2023-09-22T23:14:58.678492714Z"ip-10-150-1-55.us-west-2.compute.internal",","hostnamepid"":":4759}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759{}
"msg":"{"msg":"[2] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":[0] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) New New New ds_transition to WaitActive"","crucible"v,"":level"0:,30"name":"crucible","level":30,,""time"time":":"2023-09-22T23:14:58.678544128Z"2023-09-22T23:14:58.678547548Z",","hostname":hostname":""ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",,""pid"pid"::47594759}}

{"msg{":""msg":"[0] Transition from New to WaitActive","v":0[2] Transition from WaitActive to WaitQuorum",",name""v"::"0crucible",,""name":level":"30crucible","level":30,"time":"2023-09-22T23:14:58.67860458Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time":","pid":2023-09-22T23:14:58.678607207Z"4759,"}hostname"
:"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759"}msg":
"{"msg":"[0] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"[2] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) Active Active WaitQuorum ds_transition to Active"crucible",",v":"0level":,30"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678668054Z",",hostname":""time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:14:58.678671811Zpid"":4759,"hostname}":
"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759"msg":"}
[0] Transition from WaitActive to WaitQuorum","v":0,"name":"{crucible","level":"30msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678718932Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.678727702Z","hostname"":msg"":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[0] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) WaitQuorum New New ds_transition to Active","v":0,"{name":"crucible",""level":msg":"301cff5a3f-832b-41e6-a1da-ab29a77accd1 is now active with session: 67e248db-469f-4db7-bdb8-64747eb67344","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678770879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.678780776Z",""hostname"msg":":"[0] Transition from WaitQuorum to Active"ip-10-150-1-55.us-west-2.compute.internal",,""pid":v":47590,"}name"
:"crucible","level":30{"msg":"[1] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) Active Active Active ds_transition to Faulted","v":0,"name":","crucible"time",:""level":302023-09-22T23:14:58.678822365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{","msgtime"":":"2023-09-22T23:14:58.678840201Z","hostname":"[1] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) Active New New ds_transition to WaitActive"ip-10-150-1-55.us-west-2.compute.internal",",v"":pid0",:"4759name":"}crucible
","level":30{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678880292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}{
","timemsg""::""{2023-09-22T23:14:58.678892055Z"",Crucible stats registered with UUID: 1702975c-d4cc-4dbc-8ef2-d4f28d842baamsg""hostname":,"""v"ip-10-150-1-55.us-west-2.compute.internal:"0,",pid"":name4759":}":crucible
"","level{":30[1] Transition from New to WaitActive""msg":","v":0,"name":"crucible","level":[1] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) Active Faulted Active ds_transition to LiveRepairReady"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.678927721Z",","hostname":time","":"time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:14:58.678936415Z"pid2023-09-22T23:14:58.678941423Z"":,4759","}hostname"
:"hostname":"{ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid""msg:":4759""}
,"pid":{Crucible 1702975c-d4cc-4dbc-8ef2-d4f28d842baa has session id: be7e299b-d8c8-4183-a620-a3152a3ad667"4759","msg"v:"":}0
[1] Transition from Faulted to LiveRepairReady,"",name""v:"":0crucible{,"",name"":"level""crucible:"30,"msg":level":"30[1] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) Active WaitActive New ds_transition to WaitQuorum","v",":time0",",:""time":name":2023-09-22T23:14:58.678990323Z""","2023-09-22T23:14:58.67899399Zcrucible"hostname"",:"",hostname""ip-10-150-1-55.us-west-2.compute.internal:level"","":pidip-10-150-1-55.us-west-2.compute.internal"":,4759"30}pid"
:4759}
{{"msg"":"msg":"[0] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) New New New ds_transition to WaitActive,"[1] 1cff5a3f-832b-41e6-a1da-ab29a77accd1 (67e248db-469f-4db7-bdb8-64747eb67344) Active LiveRepairReady Active ds_transition to LiveRepair,"","v"timev""::00,,""":namename""::"""cruciblecrucible"",,""2023-09-22T23:14:58.67902287Z"levellevel",:"30:"30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,,""timetime""::""{2023-09-22T23:14:58.679054825Z2023-09-22T23:14:58.679053703Z"",,""hostname""msg":hostname""::"ip-10-150-1-55.us-west-2.compute.internal"","ip-10-150-1-55.us-west-2.compute.internalpid"":,"4759pid"[1] Transition from WaitActive to WaitQuorum":}
4759,"}{
"vmsg":{"":0","[1] Transition from LiveRepairReady to LiveRepairmsg"":name":"","crucible"v[0] Transition from New to WaitActive"":,0",v,"":name"0:,"""namecrucible"":,""levellevelcrucible"":,30"":level":3030,"time":"2023-09-22T23:14:58.679113698Z,"",time"":"hostname":"2023-09-22T23:14:58.679116886Z","ip-10-150-1-55.us-west-2.compute.internal"hostname",:""pid",:"4759ip-10-150-1-55.us-west-2.compute.internal"},
"timepid""::"4759}
2023-09-22T23:14:58.679116552Z","{hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[0] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) WaitActive New New ds_transition to WaitQuorum","v":0{,"name":""crucible"msg,":""level":30,"time":"2023-09-22T23:14:58.679165497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.679202035Z",[1] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) Active WaitQuorum New ds_transition to Active""hostname,"":v"":0,"name":ip-10-150-1-55.us-west-2.compute.internal"","cruciblepid":"4759,"level}"
:30{"msg":"[0] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible",,""leveltime""::"302023-09-22T23:14:58.679236515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,""time":"msg":"2023-09-22T23:14:58.679256014Z","[1] Transition from WaitQuorum to Active"hostname":","v":0,"ip-10-150-1-55.us-west-2.compute.internalname":""crucible,""pid",:"4759level":30}
{"msg,":""time":"[0] Transition from WaitQuorum to Active","2023-09-22T23:14:58.679288268Z"v,":"0hostname",":"name":"crucible",ip-10-150-1-55.us-west-2.compute.internal"",level"":pid"30:4759}
{"msg":"[2] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) Active Active New ds_transition to WaitActive","v":0,","name":time"":"crucible","2023-09-22T23:14:58.67931569Z"level":,"30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.679339294Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) Active New New ds_transition to WaitActive","{v":0",msg"":"name":"crucible"[2] Transition from New to WaitActive,""level",:"30v":0,"name":"crucible","level":30,,""time"time":":"2023-09-22T23:14:58.679374175Z"2023-09-22T23:14:58.679378666Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internaltest live_repair::repair_test::test_live_repair_deps_flush_repair_flush ... ",ok"pid
":4759ip-10-150-1-55.us-west-2.compute.internal","}pid
":4759}
{{"msg"":"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level"[2] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) Active Active WaitActive ds_transition to WaitQuorum":,30"v":0,"name":"crucible","level":30,",time"":"time":"2023-09-22T23:14:58.679435343Z",2023-09-22T23:14:58.679440037Z"","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",,""pid"pid"::47594759}
}
{"msg":"{[2] Transition from WaitActive to WaitQuorum"",msg":""v":0,"name":"crucible"[1] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) Active WaitActive New ds_transition to WaitQuorum",","v":level"0:,"30name":"crucible","level":30,"time":","time":"2023-09-22T23:14:58.679493197Z"2023-09-22T23:14:58.679495224Z",","hostname":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}ip-10-150-1-55.us-west-2.compute.internal"
,"pid":4759{}
"msg":"{[1] Transition from WaitActive to WaitQuorum","v":"0,msg""name":":crucible"","level":30[2] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) Active Active WaitQuorum ds_transition to Active","v":0,","nametime":"":"2023-09-22T23:14:58.67953667Z"crucible",",hostname":""level":ip-10-150-1-55.us-west-2.compute.internal30","pid":4759}
{"msg":"[1] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) Active WaitQuorum New ds_transition to Active",,""v":time"0:,""name":"crucible","level2023-09-22T23:14:58.679553041Z"":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.67957117Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[2] Transition from WaitQuorum to Active",{"v"":msg0":","name":[1] Transition from WaitQuorum to Active"","v":crucible"0,,""name":level"":crucible"30,"level":30,"time":"2023-09-22T23:14:58.679605946Z",,""hostname":time"":"ip-10-150-1-55.us-west-2.compute.internal","pid2023-09-22T23:14:58.679605162Z"":4759,"}
hostname":"{"msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[2] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) Active Active New ds_transition to WaitActive","v":0,"name":"{crucible","level":30"msg":","timeb0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 is now active with session: 1a550ef9-f088-45b5-a367-3199209f5bd1"":","v"2023-09-22T23:14:58.679636751Z":,0","hostname":"name":"ip-10-150-1-55.us-west-2.compute.internalcrucible"",,""pid":level":475930}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time",:""time":"2023-09-22T23:14:58.679666001Z"2023-09-22T23:14:58.679658071Z",",hostname"":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
ip-10-150-1-55.us-west-2.compute.internal",{"pid"":msg4759":"}
[2] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible"{,"level":30"msg":","time":"2023-09-22T23:14:58.679696742Z"[1] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) Active Active Active ds_transition to Faulted",","hostname":"v":0ip-10-150-1-55.us-west-2.compute.internal",",pid":"4759name}"
:"{crucible"","msg":"level":30[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67972546Z","hostname":",ip-10-150-1-55.us-west-2.compute.internal"","time"pid"::"4759}
2023-09-22T23:14:58.679722289Z"{,""hostnamemsg":"":"ip-10-150-1-55.us-west-2.compute.internal","[2] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) Active Active WaitQuorum ds_transition to Active"pid",":v4759":0,"name":"}crucible"
,"level":30{"msg":","time":"[1] Transition from Active to Faulted"2023-09-22T23:14:58.679761194Z",,""hostnamev":"0:","name":"ip-10-150-1-55.us-west-2.compute.internal","crucible"pid":,4759"}level"
:30{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.679788773Z,"","timehostname":"":"ip-10-150-1-55.us-west-2.compute.internal","pid":2023-09-22T23:14:58.679783345Z"4759,}
"hostname":{""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":47591702975c-d4cc-4dbc-8ef2-d4f28d842baa is now active with session: baac46cc-c0a4-4a68-b7fe-469cb936213f","}v":0
,"name":"crucible","level":30{"msg":","time":"2023-09-22T23:14:58.679822704Z","hostname":"[1] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) Active Faulted Active ds_transition to LiveRepairReady"ip-10-150-1-55.us-west-2.compute.internal",,""vpid":":47590}
,"name{":""msgcrucible"":","level":30[1] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.679851484Z",","hostname"time":":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.679847003Z,""pid":,4759"}hostname"
:"{ip-10-150-1-55.us-west-2.compute.internal","pid":"4759msg":"}
[1] Transition from Active to Faulted","v":0{,"name":"crucible"","msg"level"::"30[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.67989964Z","hostname":,""time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.679907111Z,""pid":,4759"hostname}"
:"{ip-10-150-1-55.us-west-2.compute.internal","pid":"4759msg":"}
[1] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) Active Faulted Active ds_transition to LiveRepairReady{","v":0","msgname":""crucible":,""level":30[1] b0d4bfe3-554b-45f1-b2c7-a87fcc965ee5 (1a550ef9-f088-45b5-a367-3199209f5bd1) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,","name"time":":"2023-09-22T23:14:58.67995029Z"crucible,""hostname":,""level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible",",level"":30time":"2023-09-22T23:14:58.679968358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal,""time,":""2023-09-22T23:14:58.67998093Z"pid,""hostname:":"4759}ip-10-150-1-55.us-west-2.compute.internal"
,"pid":4759}
{{""msg":msg":""[1] Transition from LiveRepairReady to LiveRepair","[1] 1702975c-d4cc-4dbc-8ef2-d4f28d842baa (baac46cc-c0a4-4a68-b7fe-469cb936213f) Active LiveRepairReady Active ds_transition to LiveRepair"v",":v":00,",name":""crucible"name",":level":"30crucible","level":30,"time":"2023-09-22T23:14:58.680025966Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time,"":"pid":47592023-09-22T23:14:58.680028527Z}"
,"hostname":"{"ip-10-150-1-55.us-west-2.compute.internal"msg{":,""pid"":msg[1] Transition from LiveRepairReady to LiveRepair"":","4759v":}0Crucible stats registered with UUID: 9389be7a-e733-4e8f-a375-dbab99314661,""
name,"":"v":crucible"0,","level"name:":30"crucible","level":30,"time":"2023-09-22T23:14:58.680071575Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal:"","pid":47592023-09-22T23:14:58.680074242Z"},
"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 9389be7a-e733-4e8f-a375-dbab99314661 has session id: 51553c06-313b-486e-a473-4e2fb6b5da09","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680119242Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680158624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680195698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680236531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30test live_repair::repair_test::test_live_repair_deps_mix ... ok,"
time":"2023-09-22T23:14:58.680274506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68030501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68032395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"test live_repair::repair_test::test_live_repair_deps_no_overlap ... [1] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) Active New New ds_transition to WaitActive"ok,
"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680365339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680404698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680445931Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680484155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680523945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680560239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68059973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680635314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680670335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68070579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680746347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680784422Z","{hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759"}
msg":"{Crucible stats registered with UUID: e55b2cbb-f658-4df1-a2c6-bb10a7952e80""msg":","v":0,"name":"9389be7a-e733-4e8f-a375-dbab99314661 is now active with session: 2b861982-700b-4629-9a70-6e31dee46671crucible"",",level"":v":300,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680829561Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal":,""pid":47592023-09-22T23:14:58.680825827Z"}
,"hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid""msg"::4759"}
[1] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) Active Active Active ds_transition to Faulted","v":0,"name":"{crucible","level":30"msg":"Crucible e55b2cbb-f658-4df1-a2c6-bb10a7952e80 has session id: 26164dc7-fc5c-47ae-b2e3-73a50f49b4e3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680882958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.680894064Z"","msg":hostname"":"[1] Transition from Active to Faulted"ip-10-150-1-55.us-west-2.compute.internal,"",v""pid"::47590,"}name
":"crucible","level":30{"msg":"[0] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) New New New ds_transition to WaitActive","v":,"0time":","name":"2023-09-22T23:14:58.680933367Z"crucible",","hostnamelevel""::30"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"{time":""2023-09-22T23:14:58.680955525Z"msg":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) Active Faulted Active ds_transition to LiveRepairReady"},
"v":0,"name":"crucible"{,"level":30"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.680994116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.681004984Z{","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","[1] Transition from Faulted to LiveRepairReadypid"":,"4759v":0},
"name":"crucible","level":{30"msg":"[0] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.681047316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.681061556Z",""hostname":"msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] 9389be7a-e733-4e8f-a375-dbab99314661 (2b861982-700b-4629-9a70-6e31dee46671) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"{crucible","level":30"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.681100963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.681110421Z","hostname""{msg":":""msg[1] Transition from LiveRepairReady to LiveRepair"":"ip-10-150-1-55.us-west-2.compute.internal",,""Crucible stats registered with UUID: ce2db535-1138-455f-9805-280fa31ce378v"":pid",0",v""name:"0:,"":namecrucible"":","cruciblelevel"",:"304759level":30}
{,""time":"msg":"2023-09-22T23:14:58.681157801Z",,""time"hostname"::""2023-09-22T23:14:58.68115977Z"ip-10-150-1-55.us-west-2.compute.internal,"",[0] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) WaitQuorum New New ds_transition to Active"hostname"":pid,""":v"4759ip-10-150-1-55.us-west-2.compute.internal":,}"
pid"0:4759,"name":"}
crucible","level":{30"msg":"Crucible ce2db535-1138-455f-9805-280fa31ce378 has session id: 44873b2a-74af-4682-8e7e-bc088fb77067","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.681203553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid",":time"4759:"}
2023-09-22T23:14:58.681215832Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759"}msg
":"[0] Transition from WaitQuorum to Active"{,"v":0","msg"name":":"crucible","level":30[0] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.681262188Z","hostname":,""time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.681270436Z","pid,"":hostname4759":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""msg":"[0] Transition from New to WaitActive","[1] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) Active New New ds_transition to WaitActivev":"0,,""namev""::"0crucible",,""level"name":":crucible"30,"level":30,"time":"2023-09-22T23:14:58.681328876Z",",hostname"":"time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.681330776Z,""pid":4759,"hostname"}:
"ip-10-150-1-55.us-west-2.compute.internal","pid"{:4759}"
msg":"{[0] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) WaitActive New New ds_transition to WaitQuorum""msg",:""v":0[1] Transition from New to WaitActive",","name"v":":crucible"0,,""namelevel"":":crucible"30,"level":30,"time":"2023-09-22T23:14:58.681396445Z",",time"":hostname":""ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.681398257Z,""pid":4759,"hostname"}:
"ip-10-150-1-55.us-west-2.compute.internal","{pid":4759"}msg":
"[0] Transition from WaitActive to WaitQuorum","v"{:0,"name":"crucible""msg":,""level":30[1] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucibletest live_repair::repair_test::test_live_repair_deps_reads ... ",ok
,""leveltime":"":302023-09-22T23:14:58.681454456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68148364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",","timepid"":":47592023-09-22T23:14:58.681499838Z","}hostname":
"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""msg":"[1] Transition from WaitActive to WaitQuorum"[0] Transition from WaitQuorum to Active",",v"":v":00,,""name"name:"":crucible"","cruciblelevel"":,"30level":30,"time":"2023-09-22T23:14:58.681549288Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal":,""pid":47592023-09-22T23:14:58.681550468Z"}
,"hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759"msg":"}
[1] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) Active New New ds_transition to WaitActive","v":0,"name":"{crucible","level":30"msg":"[1] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.681594939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.681607867Z"","msg":hostname":""ip-10-150-1-55.us-west-2.compute.internal"[1] Transition from New to WaitActive",",pid"":v":47590,"name}"
:"crucible","level":30{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.681650374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.68166212Z","hostname"":msg"":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) Active WaitActive New ds_transition to WaitQuorum","v":0,"{name":"crucible",""level"msg:"30:"[2] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.681704421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.681715783Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":[1] Transition from WaitActive to WaitQuorum4759","}v":
0,"name":"crucible","level"{:30"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.681754173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time{":"2023-09-22T23:14:58.681765905Z""msg":,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) Active WaitQuorum New ds_transition to Active"}
,"v":0,"name":"{crucible","level":30"msg":"[2] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68180374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,{"time":""msg":"2023-09-22T23:14:58.681816857Z","hostname":[1] Transition from WaitQuorum to Active"","v":0ip-10-150-1-55.us-west-2.compute.internal",,""pid"name"::4759"crucible"},
"level":30{"msg":"[2] Transition from WaitActive to WaitQuorum","v":,"0time",:""name":"2023-09-22T23:14:58.681854969Z"crucible",","hostname":"level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":[2] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) Active Active New ds_transition to WaitActive"","v":02023-09-22T23:14:58.681875563Z","name":","crucible"hostname,""level"::"30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,""timemsg":"":"2023-09-22T23:14:58.681903757Z","hostname":"[2] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) Active Active WaitQuorum ds_transition to Activeip-10-150-1-55.us-west-2.compute.internal"","pid",":v"4759:0},"
name":"crucible","level":30{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68194846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.681960757Z"{,"hostname":""msg"ip-10-150-1-55.us-west-2.compute.internal":{","pid":4759[2] Transition from WaitQuorum to Active"},
""v"msg":{:0",""name"msg"::Crucible stats registered with UUID: 8eddf9f4-e32a-4516-96f5-18a954e25241"""crucible",,""v"level:[2] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) Active Active WaitActive ds_transition to WaitQuorum"0",:,30""namev":":"0crucible,"",name":""level":crucible"30,"level":30,"time":"2023-09-22T23:14:58.682017617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.68203163Z",","time""hostname":"msg":":ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.682029986Z,""",pid"":hostname4759e55b2cbb-f658-4df1-a2c6-bb10a7952e80 is now active with session: 10373868-88fa-495a-b411-62c3231f0270""}:
","v":0{,"ip-10-150-1-55.us-west-2.compute.internal"name",""pidmsg"":"::crucible""4759,"level":[2] Transition from WaitActive to WaitQuorum"30},
"v":0,"name":"crucible","{level":30"msg":","time":"Crucible 8eddf9f4-e32a-4516-96f5-18a954e25241 has session id: a10d4b49-4ccf-45c2-853d-34a55302e6802023-09-22T23:14:58.682092229Z"",,""hostnamev,"":"0:,"time"":ip-10-150-1-55.us-west-2.compute.internal"name,""":"pidcrucible"2023-09-22T23:14:58.682104914Z"":,4759","hostname"level:}"
":30ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"msg":"{"msg":"[1] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) Active Active Active ds_transition to Faulted",",time"":v":[2] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) Active Active WaitQuorum ds_transition to Active"0,",name"":"v":crucible0",",name":""cruciblelevel"":,"30level":30"2023-09-22T23:14:58.682148231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":",{"time"2023-09-22T23:14:58.682183867Z":"","msg2023-09-22T23:14:58.68218282Z"":",hostname":""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pidip-10-150-1-55.us-west-2.compute.internal"[0] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) New New New ds_transition to WaitActive"",:,""vpid":47594759":}0}

,{"name":"crucible",{""msglevel""":msg":"":30[1] Transition from Active to Faulted"[2] Transition from WaitQuorum to Active",","vv""::00,,""namename"":":"crucible"crucible",,,""level"time"::"30"level":2023-09-22T23:14:58.682251102Z30","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":",",time[0] Transition from New to WaitActive""",:"time":""2023-09-22T23:14:58.682274192Z"v":,02023-09-22T23:14:58.682276895Z,""hostname",:"""name":ip-10-150-1-55.us-west-2.compute.internal"","hostname":"crucible"pid",ip-10-150-1-55.us-west-2.compute.internal":",level":"304759pid":4759}
}
{,"time":""{2023-09-22T23:14:58.682324192Z"msg":",""hostname":"msg":"ce2db535-1138-455f-9805-280fa31ce378 is now active with session: cf305ba7-b9a0-443c-94d6-97299bd8db90"ip-10-150-1-55.us-west-2.compute.internal",,""v"pid:"0:[1] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) Active Faulted Active ds_transition to LiveRepairReady",",name"4759:"}"
v":crucible0"{,,""level"name"":msg":"":crucible"30,"level":30[0] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":",",time":"2023-09-22T23:14:58.682377278Z""2023-09-22T23:14:58.68237907Z"time",:,"""hostname"hostname"2023-09-22T23:14:58.682386822Z":":","ip-10-150-1-55.us-west-2.compute.internalhostnameip-10-150-1-55.us-west-2.compute.internal""",,:"""pid"pid"ip-10-150-1-55.us-west-2.compute.internal:":,47594759"pid}}"

:{4759}
"{msg":"{""msg"msg"::[1] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) Active Active Active ds_transition to Faulted""","v":0[0] Transition from WaitActive to WaitQuorum,[1] Transition from Faulted to LiveRepairReady"""name",:"",v"":v":crucible"00,,""level","name"name":::""30crucible"crucible",",level":"30level":30,,""time":time"":"2023-09-22T23:14:58.682490893Z"2023-09-22T23:14:58.682484953Z,,"""hostname",":hostnametime"":"":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.682490231Z"ip-10-150-1-55.us-west-2.compute.internal",,",pid"":"pid"hostname"::47594759"}
ip-10-150-1-55.us-west-2.compute.internal}"
{,"pid":4759"msg":{"}
[0] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) WaitQuorum New New ds_transition to Active"",msg":""{v"[1] Transition from Active to Faulted":,"0",v":msg"0",name":name""::""crucible"","cruciblelevel"":,"30level"[1] e55b2cbb-f658-4df1-a2c6-bb10a7952e80 (10373868-88fa-495a-b411-62c3231f0270) Active LiveRepairReady Active ds_transition to LiveRepair":30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.682578931Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal":","pid":47592023-09-22T23:14:58.682581295Z"}
,","hostname"{:time""":msg":"ip-10-150-1-55.us-west-2.compute.internal"","pid":2023-09-22T23:14:58.682587359Z4759[1] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) Active Faulted Active ds_transition to LiveRepairReady"","}v,"
hostname""::"{0,"name":"crucible"ip-10-150-1-55.us-west-2.compute.internal"",msg","level":pid30""::"4759}[0] Transition from WaitQuorum to Active"
,,""time":"v":{2023-09-22T23:14:58.682635081Z"0,","hostname":"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible"","msg":",pid":"4759[1] Transition from LiveRepairReady to LiveRepair"level","}v"::030
,"name":"{crucible""msg":","level":[1] Transition from Faulted to LiveRepairReady"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.682671892Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time,"":"pid":47592023-09-22T23:14:58.682684234Z","}hostname":","
time"ip-10-150-1-55.us-west-2.compute.internal":,""{pid":47592023-09-22T23:14:58.682681791Z"}",
"msg{"hostname":":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}[1] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) Active New New ds_transition to WaitActive[1] ce2db535-1138-455f-9805-280fa31ce378 (cf305ba7-b9a0-443c-94d6-97299bd8db90) Active LiveRepairReady Active ds_transition to LiveRepair"
","v":0,","v"name":":crucible"0,",level":30"name":"crucible","level":30,"time":"2023-09-22T23:14:58.682742197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v",":0time",":name":""crucible","level":302023-09-22T23:14:58.682746606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,"4759time":"}2023-09-22T23:14:58.6827666Z"
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","{pid":4759}
"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.682796219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.682836967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.682870941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.682910806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.682947919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) Active Active New ds_transition to WaitActive","v":0,"name":"crucibletest live_repair::repair_test::test_live_repair_deps_repair_flush_repair ... "ok
,"level":30,"time":"2023-09-22T23:14:58.683025957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683065844Z","hostname":"test live_repair::repair_test::test_live_repair_deps_repair_flush ... ip-10-150-1-55.us-west-2.compute.internalok
","pid":4759}
{"msg":"[2] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683114072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683149014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683214803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68325313Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"8eddf9f4-e32a-4516-96f5-18a954e25241 is now active with session: 7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683289866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683330424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683367768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683406197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683437759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8eddf9f4-e32a-4516-96f5-18a954e25241 (7c3b4d36-4dbe-4df1-b74a-ee41370cd9e4) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68347244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683506691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""Write to Extent 1:2:9 under repair"msg,""v":":0,"name":"crucible","Crucible stats registered with UUID: be858f8c-b2ed-4658-ae55-df51207082cblevel"":40,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683549173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.683556758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""Crucible be858f8c-b2ed-4658-ae55-df51207082cb has session id: 7c583b93-3217-4cbe-8866-c7e99e1310b0"msg",":v"":0,"name":"crucible"Crucible stats registered with UUID: 7070e84b-45e9-4b98-b03f-656a6ba86de7,""level":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683620584Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.683626941Z"{,"hostname":""ip-10-150-1-55.us-west-2.compute.internal"msg",":pid"":4759}
[0] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) New New New ds_transition to WaitActive","v":0,"{name":"crucible",""level"msg"::"30Crucible 7070e84b-45e9-4b98-b03f-656a6ba86de7 has session id: 65505822-2360-4a3c-803c-47f082ed4b20","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683688224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.683697918Z",""hostname":"msg":"ip-10-150-1-55.us-west-2.compute.internal"[0] Transition from New to WaitActive",","pid"v"::47590,"name":}"
crucible","level":30{"msg":"[0] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683741075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.683756719Z",""hostnamemsg"":":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[0] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) WaitActive New New ds_transition to WaitQuorum","v":0,"{name":"crucible","level":"30msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683797028Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time",":"pid":47592023-09-22T23:14:58.683805054Z"},
"hostname":"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759"msg":"}
[0] Transition from WaitActive to WaitQuorum","v":0,"name"{:"crucible","level":"30msg":"[0] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68385249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.683864342Z"{,"hostname":""msg"ip-10-150-1-55.us-west-2.compute.internal":","pid":4759}
[0] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) WaitQuorum New New ds_transition to Active","v":0{,"name":"crucible"",msg"":"level":30[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683907676Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time":","pid":2023-09-22T23:14:58.683914555Z"4759,"}hostname"
:"ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"msg":"[0] Transition from WaitQuorum to Active"{,"v":0,""msg":"name":"crucible","level":30[0] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.683964255Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal,""time,""pid"::"47592023-09-22T23:14:58.683971726Z}"
,"hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759"msg"}:
"{"[1] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) Active New New ds_transition to WaitActive"msg":","v":0[0] Transition from WaitQuorum to Active",","namevtest live_repair::repair_test::test_live_repair_deps_repair_kitchen_sink ... ":ok0
,""name:"":"crucible"crucible",",level":"30level":30,","time":"time":"2023-09-22T23:14:58.684045377Z"2023-09-22T23:14:58.684046321Z",","hostname":"hostname":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",,""pidpid""::47594759}
}
{"msg":"{[1] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) Active New New ds_transition to WaitActive",""v":msg0",:""name":"crucible","level[1] Transition from New to WaitActive"":30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.684104238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":,""time":"[1] Transition from New to WaitActive","2023-09-22T23:14:58.684109802Z"v":,0","hostname":"name":"crucible","ip-10-150-1-55.us-west-2.compute.internal"level":,"30pid":4759}
,"time":"2023-09-22T23:14:58.684140613Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"msg":{""msg":"[1] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) Active WaitActive New ds_transition to WaitQuorum"[1] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) Active WaitActive New ds_transition to WaitQuorum","v":0,",v"":name":"0crucible",","namelevel":":"30crucible","level":30,"time":"2023-09-22T23:14:58.684179407Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.684182545Z","hostname":","time":"2023-09-22T23:14:58.684198784Z"ip-10-150-1-55.us-west-2.compute.internal,"",hostname":""pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid":4759}}

{{"msg":""msg":"[1] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) Active WaitQuorum New ds_transition to Active","v":0,"[1] Transition from WaitActive to WaitQuorum"name":","cruciblev":","0level":30,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.684234937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.684239246Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal",2023-09-22T23:14:58.684253645Z"","pidhostname":"":4759ip-10-150-1-55.us-west-2.compute.internal","pid":4759}}

{{"msg":""msg":"[2] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30[1] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible",",time"":"level":2023-09-22T23:14:58.684284201Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,","time":"time":"2023-09-22T23:14:58.684306009Z"2023-09-22T23:14:58.684295611Z","hostname":","hostnameip-10-150-1-55.us-west-2.compute.internal","":pid":"4759}
ip-10-150-1-55.us-west-2.compute.internal","{pid":"4759msg":"}
[2] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30{"msg":"[1] Transition from WaitQuorum to Active,""time":","v"2023-09-22T23:14:58.684338122Z":,"0hostname",:""name":"ip-10-150-1-55.us-west-2.compute.internal","cruciblepid":"4759,"}level":
30{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.684363217Z","hostname":","timeip-10-150-1-55.us-west-2.compute.internal"",":"pid":47592023-09-22T23:14:58.68437373Z","hostname}":"
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""msg":"[2] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","[2] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) Active Active New ds_transition to WaitActivelevel":"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.684415318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.684420145Z"",msg"":"hostname":"[2] Transition from WaitQuorum to Active",ip-10-150-1-55.us-west-2.compute.internal"","v"pid"::04759,"name":"}crucible
","level":30{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible",","leveltime":"":302023-09-22T23:14:58.684462483Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"7070e84b-45e9-4b98-b03f-656a6ba86de7 is now active with session: f1f39d9e-3c69-4dbe-901b-424816224ade","v":0,"name":"crucible","level":,30"time":"2023-09-22T23:14:58.684478737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",",time":""pid"2023-09-22T23:14:58.684496615Z:"4759,"hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"{msg":""msg":"[2] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) Active Active WaitActive ds_transition to WaitQuorum","v"[1] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) Active Active Active ds_transition to Faulted":,"0v":,0","name"name{":":"crucible"crucible"",","msglevel""::"30level":30Crucible stats registered with UUID: e2411e4d-ee59-47a2-bfc3-972c6325e258","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.684550675Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal":,""pid":47592023-09-22T23:14:58.684552206Z"}
,",time"{:"""hostname"msg:"2023-09-22T23:14:58.684560327Z"":","ip-10-150-1-55.us-west-2.compute.internal"[1] Transition from Active to Faultedhostname,"":",""pidvip-10-150-1-55.us-west-2.compute.internal"":,"0",pid""name:"4759:":crucible}"
,"4759level":30{}
"msg":"{,"Crucible e2411e4d-ee59-47a2-bfc3-972c6325e258 has session id: 2a5689ec-7f51-4a8b-9e84-e207c2db9743"time":","v""2023-09-22T23:14:58.684604515Z:"0msg":",,""namehostname""::[2] Transition from WaitActive to WaitQuorum""","crucible"ip-10-150-1-55.us-west-2.compute.internal,""vlevel,"":pid30":"4759:}0
,"name":"{crucible,"""msg,"time:"":""level":2023-09-22T23:14:58.684629502Z"30,"hostname":"[1] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) Active Faulted Active ds_transition to LiveRepairReady","ip-10-150-1-55.us-west-2.compute.internalv"",:"0pid,"":4759name":"}
crucible","level"{:30"msg":"[0] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) New New New ds_transition to WaitActive",,"",vtime""":time0",:""name":":2023-09-22T23:14:58.684656533Z""crucible,"",2023-09-22T23:14:58.684646725Z"hostname"":"level,":ip-10-150-1-55.us-west-2.compute.internal30"","hostname":"pid":4759}
ip-10-150-1-55.us-west-2.compute.internal","pid"{,""time"msg:"":":2023-09-22T23:14:58.684680657Z4759"[1] Transition from Faulted to LiveRepairReady",","hostnamev""::"0},"
ip-10-150-1-55.us-west-2.compute.internalname"":,""pidcrucible"":,4759"level"}:
30{{"msg"":msg"":"[0] Transition from New to WaitActive","v,"":0time",:""name":"[2] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) Active Active WaitQuorum ds_transition to Active2023-09-22T23:14:58.684711734Zcrucible""",,"",levelhostname"":"30:"v":0,ip-10-150-1-55.us-west-2.compute.internal"","name":"pid"crucible:,4759""}time,":
""level":2023-09-22T23:14:58.684730791Z{"30,""msg":hostname"":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] 7070e84b-45e9-4b98-b03f-656a6ba86de7 (f1f39d9e-3c69-4dbe-901b-424816224ade) Active LiveRepairReady Active ds_transition to LiveRepair","v{":0,""namemsg""::""crucible","level":30[0] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) WaitActive New New ds_transition to WaitQuorum","v":,"0time,"",name"":"time"crucible:":,"""level"2023-09-22T23:14:58.684760777Z:"30,2023-09-22T23:14:58.684746911Z""hostname":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pidip-10-150-1-55.us-west-2.compute.internal"",",:"4759pid"}time
":":{47592023-09-22T23:14:58.684776031Z"",msg"":hostname"":"}
[1] Transition from LiveRepairReady to LiveRepairip-10-150-1-55.us-west-2.compute.internal"",,""pidv""::47590,"}
name"{{:"crucible"",msg"":"level":"30msg[0] Transition from WaitActive to WaitQuorum"":","v":0,[2] Transition from WaitQuorum to Active""name":,""crucible,"",time""level"::"30v":2023-09-22T23:14:58.684811651Z"0,","hostname"name":":"crucible",ip-10-150-1-55.us-west-2.compute.internal,"","time"pid:"":"47592023-09-22T23:14:58.684825306Z"},
"level"hostname"::"30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) WaitQuorum New New ds_transition to Active","v":0,",name":""crucible"time":","level":2023-09-22T23:14:58.684845033Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.684869216Z"{,"hostname":""msg":ip-10-150-1-55.us-west-2.compute.internal"","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name"be858f8c-b2ed-4658-ae55-df51207082cb is now active with session: e05bbd9e-fb85-4807-b00a-a7affda3fe87:""crucible","level":30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.684957595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.684966876Z",",time":""hostname2023-09-22T23:14:58.684985327Z"":,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid":"4759,}"
pid":{4759"msg":"}
[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30{"msg":","time":"2023-09-22T23:14:58.685019073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) Active Active Active ds_transition to Faulted"{,"v":"0msg":","name":"crucible","level[1] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) Active WaitActive New ds_transition to WaitQuorum"":,"v30":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685047245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",",pid":"4759time}"
:"{2023-09-22T23:14:58.685045079Z",""msg":"hostname":"[1] Transition from WaitActive to WaitQuorum"ip-10-150-1-55.us-west-2.compute.internal,"",v"":pid":04759,"name":"}
crucible","level":30{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:14:58.685088151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,""msg":"time":"2023-09-22T23:14:58.685099476Z","[1] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) Active WaitQuorum New ds_transition to Active"hostname":","v":0,"name":"crucible"ip-10-150-1-55.us-west-2.compute.internal",",level":"30pid":4759}
{"msg":","time":"2023-09-22T23:14:58.685125188Z"[1] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) Active Faulted Active ds_transition to LiveRepairReady",","hostname":v"":0,"nametest live_repair::repair_test::test_live_repair_deps_repair_overlappers ... ip-10-150-1-55.us-west-2.compute.internal"",ok"pid
"::4759"crucible"},"
level":30{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685222984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level",:"30time":"2023-09-22T23:14:58.685166802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid,""time"::"47592023-09-22T23:14:58.68526541Z",}"
hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""msg":[2] Transition from New to WaitActive"","v":0,"name":"crucible",[1] Transition from Faulted to LiveRepairReady""level":30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685300691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) Active Active WaitActive ds_transition to WaitQuorum","v":0,,""name":"time":crucible"","level":302023-09-22T23:14:58.685307231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,"4759time":"}2023-09-22T23:14:58.685326071Z
","hostname":"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759}
"msg{":""msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30[1] be858f8c-b2ed-4658-ae55-df51207082cb (e05bbd9e-fb85-4807-b00a-a7affda3fe87) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible",",time"":"level":2023-09-22T23:14:58.685357854Z30","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) Active Active WaitQuorum ds_transition to Active","v":0,"name,"":"timecrucible"",":level":30"2023-09-22T23:14:58.685368687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",",time":""pid":2023-09-22T23:14:58.685386228Z"4759,"hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg"":"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"[1] Transition from LiveRepairReady to LiveRepair"crucible",,""v"level":30:0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685424378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":e2411e4d-ee59-47a2-bfc3-972c6325e258 is now active with session: d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09"","v":0,"name":"2023-09-22T23:14:58.685429243Zcrucible"",",level":"30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.685450085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685480331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68551387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685548888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685569962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2411e4d-ee59-47a2-bfc3-972c6325e258 (d6d84fee-a1eb-4f10-bf73-cb08f9fa6e09) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685590617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68560905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6","v":0,"name":"crucible","level":30test live_repair::repair_test::test_live_repair_deps_repair_other ... ok
,"time":"2023-09-22T23:14:58.685676398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 has session id: 8a93fbe4-dffc-4ef3-ac6b-8ce0395f2496","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685723932Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685765742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive"test live_repair::repair_test::test_live_repair_deps_repair_rafter ... ,"okv"
:0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685815273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685858633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685896297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685936574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.685976733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.686017785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{""msg":"msg":"[1] Transition from New to WaitActive","v":0,"Crucible stats registered with UUID: 8d04c4f4-46e4-45cb-86ab-55f2298edbab"name":"crucible",,""level"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.686072987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759:30}
{"msg":","Crucible 8d04c4f4-46e4-45cb-86ab-55f2298edbab has session id: 51715fcf-9888-475e-af8d-25850d80ded5"time",":"v":02023-09-22T23:14:58.686106554Z","name",:""hostnamecrucible"":,""level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":"2023-09-22T23:14:58.686137641Z"[1] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) Active WaitActive New ds_transition to WaitQuorum",",hostname"":v"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
":0{,"name":"crucible",""level":msg"30:"[0] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.686190857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.686202839Z","hostname"":msg":""ip-10-150-1-55.us-west-2.compute.internal"[1] Transition from WaitActive to WaitQuorum",",pid""v":4759}
:{0,"name":"crucible"","msg"level":"[0] Transition from New to WaitActive","v"{:0,"name":"crucible""msg":","level":30Crucible stats registered with UUID: 7d29f824-b397-4b45-b288-49fd0e9a3aeb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.68627544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"}
{:30"msg":"[0] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.686313667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.686327Z",""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}time
msg":"{[1] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) Active WaitQuorum New ds_transition to Active"",msg"":"v":0,"name"[0] Transition from WaitActive to WaitQuorum","v":0,"name":""crucible":","level":2023-09-22T23:14:58.686282794Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759:}
"crucible","{level":30"msg":","time":"2023-09-22T23:14:58.686392926Z"Crucible 7d29f824-b397-4b45-b288-49fd0e9a3aeb has session id: 8b5fa56a-1c47-4d69-9152-184aa57abe96",,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.686445129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":""msg":"[0] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) WaitQuorum New New ds_transition to Active"[0] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) New New New ds_transition to WaitActive",","vv",:""0:time0,"name":"crucible",""level"::"302023-09-22T23:14:58.686407543Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"name",:""pid":crucible"4759,"level"}:
30{,"time"":msg"":"2023-09-22T23:14:58.686492573Z"[1] Transition from WaitQuorum to Active,"",hostname,""v"::0,"name":"crucible",""level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.686528424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30{,"time":""2023-09-22T23:14:58.686561501Z"msg":,""hostname":"[0] Transition from WaitQuorum to Active"ip-10-150-1-55.us-west-2.compute.internal",,"""timepid""::"v4759"2023-09-22T23:14:58.686505412Z"}
,:"0hostname"{,"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.686610127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"":4759name":}"
crucible",":{level""":msgip-10-150-1-55.us-west-2.compute.internal""30:,""pid":4759}
[2] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) Active Active WaitActive ds_transition to WaitQuorum","v":0,"{name":"crucible",""level"msg:":30"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":",2023-09-22T23:14:58.686642889Z","time":"2023-09-22T23:14:58.686660712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":"4759time":"},
{2023-09-22T23:14:58.686653519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30"msg":"[1] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) Active New New ds_transition to WaitActive","v":,""time"time"::""0,"2023-09-22T23:14:58.686764996Z2023-09-22T23:14:58.686696682Z""name":,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30"crucible","level":30,"time":"2023-09-22T23:14:58.686813637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":""msg":"2023-09-22T23:14:58.686819998Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"hostname[0] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) WaitQuorum New New ds_transition to Active,","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.686855351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"pid":{4759"msg":}"
"{:"[0] Transition from WaitQuorum to Active"ip-10-150-1-55.us-west-2.compute.internal",,"""pidmsg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.686909999Z"",":hostname"4759:"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759{}
{""msg":"[1] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30msg":","time":"[2] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) Active Active WaitQuorum ds_transition to Active"2023-09-22T23:14:58.686970634Z",,""vhostnamev":0,"name":"crucible","level":30",":time"":"2023-09-22T23:14:58.687010852Z"ip-10-150-1-55.us-west-2.compute.internal",",hostname"":pid"":4759ip-10-150-1-55.us-west-2.compute.internal","}pid"
":0,"name":"crucible","level":30{,"time":""2023-09-22T23:14:58.687062504Z"msg":","hostname":"[1] Transition from WaitActive to WaitQuorumip-10-150-1-55.us-west-2.compute.internal","pid":4759}
","v":0{,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.687098883Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",""pid"msg:":"4759}[2] Transition from WaitQuorum to Active
:4759}
{"msg":"[1] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30{"msg":""[1] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) Active WaitQuorum New ds_transition to Active","v,,"""time":":2023-09-22T23:14:58.68716495Z","hostname":"0,"ip-10-150-1-55.us-west-2.compute.internalname"",:""pid":crucible"4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:14:58.687210896Z"level,"":hostname"30:"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.687228846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","{level":30"msg":"[1] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.687277073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
v":{0,"name":"",crucible""msg,time""level"::"30":"2023-09-22T23:14:58.68729059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"[2] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) Active Active New ds_transition to WaitActive","pid,"":v4759":}0
,",time":"{"name":""crucible","level":30,"time":"2023-09-22T23:14:58.687352273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30msg":","time":[1] Transition from WaitActive to WaitQuorum"","v":2023-09-22T23:14:58.687397977Z0,""name":","crucible"hostname",:""level"2023-09-22T23:14:58.687322368Z:"30ip-10-150-1-55.us-west-2.compute.internal","pid",":hostname4759":"}
,"time":"{2023-09-22T23:14:58.687429017Z","ip-10-150-1-55.us-west-2.compute.internalhostname"":,"""pid":ip-10-150-1-55.us-west-2.compute.internal4759msg}
{"msg":"6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 is now active with session: 99b1ca7f-770d-44d0-9b20-c9a2a545ed4c","v":0,"name":"crucible","level":30":","time":"2023-09-22T23:14:58.687473631Z","hostname":"[2] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) Active Active WaitActive ds_transition to WaitQuorum"ip-10-150-1-55.us-west-2.compute.internal",,""v"pid:","pid":4759}
{0"msg",:""name":"crucible","level":30[1] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) Active WaitQuorum New ds_transition to Active"","v"::04759,"name"}:"
crucible","level":30{"msg":"[1] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) Active Active Active ds_transition to Faulted","v":0,"name":",",crucible""time,""timelevel":":30,"time":"2023-09-22T23:14:58.68759292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"2023-09-22T23:14:58.687531401Z","hostname{":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid"[1] Transition from Active to Faulted":,"v4759":0,"}name:""
{2023-09-22T23:14:58.687540193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":":2023-09-22T23:14:58.687686477Z"","crucible"hostname":","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":,""time":","time":"2023-09-22T23:14:58.687702111Z",2023-09-22T23:14:58.687711314Z[2] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) Active Active New ds_transition to WaitActive""",,""hostnamehostnamev""::0","":nameip-10-150-1-55.us-west-2.compute.internal"":,"""pidip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30":4759}
{"msg":"crucible",",time"":level":30,"time":"2023-09-22T23:14:58.687785043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg"":"2023-09-22T23:14:58.687764788Z"[2] Transition from New to WaitActive","v":,0","hostname"name:":""crucible","level":ip-10-150-1-55.us-west-2.compute.internal30","pid":4759}
,"time":"2023-09-22T23:14:58.687827849Z"{,"hostname":""ip-10-150-1-55.us-west-2.compute.internal"msg,"":pid"[1] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) Active Faulted Active ds_transition to LiveRepairReady:"4759"},
"[2] Transition from WaitQuorum to Active"v{":,0",""msg"name:"":"v":crucible"0,","level"name"[2] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) Active Active WaitActive ds_transition to WaitQuorum":,"v":0","cruciblename"":","crucible"level",":level"30::,"time":"2023-09-22T23:14:58.687879233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
30{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":","time":2023-09-22T23:14:58.687911182Z"","2023-09-22T23:14:58.687921884Z"hostname":","30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal4759","pid"},"time":"2023-09-22T23:14:58.687939757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible"
{,"level"":msg30"::"4759}
8d04c4f4-46e4-45cb-86ab-55f2298edbab is now active with session: 85964bc3-39c1-482e-87d4-de63e3780fd4","v":{0,"name":",""crucible","level":30,"time":"2023-09-22T23:14:58.688008223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"time,"pid":4759}
"{:"2023-09-22T23:14:58.687991481Z"msg":"[1] 6c58d01c-f6ba-47d5-9bbd-2d1152f57cb6 (99b1ca7f-770d-44d0-9b20-c9a2a545ed4c) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30msg":"[1] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) Active Active Active ds_transition to Faulted","time",:""v":02023-09-22T23:14:58.688074605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.688119377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}"
,"hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid":2023-09-22T23:14:58.688127799Z4759"}
,"hostname":"{ip-10-150-1-55.us-west-2.compute.internal",""pidmsg":"":4759}
{"msg":[2] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) Active Active WaitQuorum ds_transition to Active""[1] Transition from Active to Faulted",","vv""::00,","name":name"":"crucible"crucible",,""level"level"::3030,"time":,""time":"2023-09-22T23:14:58.688199662Z","2023-09-22T23:14:58.688200073Z"hostname":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid":}4759
}
test impacted_blocks::test::iblocks_blocks_iterates_over_all_blocks ... ok
{"msg":"[1] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30{,"time":""2023-09-22T23:14:58.688292653Z"msg":,""hostname":"[2] Transition from WaitQuorum to Active"ip-10-150-1-55.us-west-2.compute.internal",",v""pid":4759}
:0{,"name":"crucible"",msg""level:""[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level"::3030,"time":",test live_repair::repair_test::test_live_repair_deps_repair_read ... "oktime":"
2023-09-22T23:14:58.688360051Z"2023-09-22T23:14:58.688359171Z","hostname,":""hostname":ip-10-150-1-55.us-west-2.compute.internal"","pid":4759ip-10-150-1-55.us-west-2.compute.internal}"
,"pid":4759}{
{"msg""msg":":[1] 8d04c4f4-46e4-45cb-86ab-55f2298edbab (85964bc3-39c1-482e-87d4-de63e3780fd4) Active LiveRepairReady Active ds_transition to LiveRepair"","v":0,"name":"7d29f824-b397-4b45-b288-49fd0e9a3aeb is now active with session: 28f01602-0927-4cde-bb59-43ed57ad0042"crucible",","level"v":30:0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.688451456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":[1] Transition from LiveRepairReady to LiveRepair"","v"2023-09-22T23:14:58.688464289Z":0,",name"":"hostname"crucible":","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":"2023-09-22T23:14:58.688507069Z","[1] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) Active Active Active ds_transition to Faultedhostname"":","v":0ip-10-150-1-55.us-west-2.compute.internal,"","name"pid:"":crucible4759","}level":
30,"time":"2023-09-22T23:14:58.688599327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":{"crucible","level":30"msg":"Crucible stats registered with UUID: 1c9731f3-b505-4ad4-b45b-f22ac14fff3b","v":0,"name":"crucible","level":,30"time":"2023-09-22T23:14:58.688650795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":"[1] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) Active Faulted Active ds_transition to LiveRepairReady","v2023-09-22T23:14:58.688665809Z"":0,,""hostnamename":"":"crucible","level":ip-10-150-1-55.us-west-2.compute.internal30","pid":4759}
,"time":"2023-09-22T23:14:58.688700165Z"{,"hostname":""msg":ip-10-150-1-55.us-west-2.compute.internal"","pid":4759}
Crucible 1c9731f3-b505-4ad4-b45b-f22ac14fff3b has session id: bfa6ab39-1847-4eeb-8aa8-fd97c3e05502","v":{0,"name"":"msg":crucible"","level":30[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.688735587Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid":2023-09-22T23:14:58.688741632Z"4759,"hostname}":
"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759"msg}"
:"{"msg[0] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) New New New ds_transition to WaitActive"":","v":0,"name":"crucible","level":[1] 7d29f824-b397-4b45-b288-49fd0e9a3aeb (28f01602-0927-4cde-bb59-43ed57ad0042) Active LiveRepairReady Active ds_transition to LiveRepair"30test live_repair::repair_test::test_live_repair_deps_repair_repair_repair ... ,"okv
":0,"name":","crucible"time",:""level":2023-09-22T23:14:58.688792311Z30","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time"":msg"":"2023-09-22T23:14:58.68881173Z"[0] Transition from New to WaitActive,"",hostname""v"::0","name":"crucible"ip-10-150-1-55.us-west-2.compute.internal",",level":"30pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair",",v"":time":"0,"name"2023-09-22T23:14:58.688847128Z":","crucible"hostname":","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg,"":time"":"2023-09-22T23:14:58.688875158Z","hostname":"[0] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) WaitActive New New ds_transition to WaitQuorum","ip-10-150-1-55.us-west-2.compute.internal"v,":"0pid",:"4759name":"}crucible
","level":30,"time":"2023-09-22T23:14:58.68891197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.688955504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg"":"msg":"Crucible stats registered with UUID: a5c9346e-78da-47f3-a9ba-6cf4321f87bd"[0] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) WaitQuorum New New ds_transition to Active",",v""v"::00,,""namename""::""crucible"crucible",","level"level:":3030,"time":"2023-09-22T23:14:58.688997304Z","hostname",:""time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.688997818Z,""pid",:"4759hostname":}"
ip-10-150-1-55.us-west-2.compute.internal","pid":4759{}
{"msg":"Crucible a5c9346e-78da-47f3-a9ba-6cf4321f87bd has session id: 41805997-5765-4deb-a3cd-4db3222bd701","v":0,"name":"crucible","level":30"msg":","time":"[0] Transition from WaitQuorum to Active"2023-09-22T23:14:58.689043544Z","v,"":0hostname",:""name":"crucible",ip-10-150-1-55.us-west-2.compute.internal""level,"":pid30":4759}
{"msg":","time":"2023-09-22T23:14:58.689064136Z","hostname":"[0] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) New New New ds_transition to WaitActive","v":ip-10-150-1-55.us-west-2.compute.internal"0,,""pid"name:":4759"crucible"},
"level":30{"msg":"[1] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) Active New New ds_transition to WaitActive","v":,"0,time"":name"":"crucible"2023-09-22T23:14:58.689087542Z,"",level"":30hostname":test live_repair::repair_test::test_live_repair_deps_repair_rspan_left ... ,""okip-10-150-1-55.us-west-2.compute.internal"
,"pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30time":"2023-09-22T23:14:58.689101515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"time":}"
2023-09-22T23:14:58.689138647Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal""msg,"":"pid":4759}
{"msg":"{[0] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) WaitActive New New ds_transition to WaitQuorum",""v":msg"0:,"name":"crucible","level":30[1] Transition from New to WaitActive","v":0,"name":"crucible","level",""Crucible stats registered with UUID: b28e974a-c998-41d2-b8a0-fef8e732ae88"time",:""v":0,"2023-09-22T23:14:58.689186008Z"name":","crucible"hostname",:""level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}:
{,"time":""msg":"2023-09-22T23:14:58.689209859Z","hostname[0] Transition from WaitActive to WaitQuorum"":","v":0,ip-10-150-1-55.us-west-2.compute.internal""name,"":"pid":crucible"4759,"level"}:
30{"msg":"Crucible b28e974a-c998-41d2-b8a0-fef8e732ae88 has session id: d887034b-9c0f-4710-a9b0-98af6b46dd84","v"30:0,"name":",crucible""time,"":"level":302023-09-22T23:14:58.689241189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":""msg":"2023-09-22T23:14:58.689253953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal,"[0] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) WaitQuorum New New ds_transition to Active,"","v":0,"name":"crucible","level":30pid":4759}
{"msg":"","time":time"":"[0] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) New New New ds_transition to WaitActive2023-09-22T23:14:58.689277765Z"",",hostname""v:"ip-10-150-1-55.us-west-2.compute.internal","pid":":47590,"}
{2023-09-22T23:14:58.689250532Z""msg":","[0] Transition from WaitQuorum to Active"hostname",:""v"name:0","ip-10-150-1-55.us-west-2.compute.internal:name"":,"pid":"4759crucible","}level":
30{""msg":"crucible","level":,30"time":"[1] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) Active WaitActive New ds_transition to WaitQuorum2023-09-22T23:14:58.689329022Z"",","hostnamev""::0","name":"crucible",ip-10-150-1-55.us-west-2.compute.internal""time,":"2023-09-22T23:14:58.68933997Z""pid,"":4759hostname":"}
{"msg":","level":30[1] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":",2023-09-22T23:14:58.689373282Z""time":,""hostname":"2023-09-22T23:14:58.689378534Z","hostname"ip-10-150-1-55.us-west-2.compute.internal:"","pid":4759ip-10-150-1-55.us-west-2.compute.internal",}"
pid":4759}
{"msg":"ip-10-150-1-55.us-west-2.compute.internal"[1] Transition from New to WaitActive",","pidv""::47590,"}name
":"crucible"{,"level":30"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.689416372Z","{hostname":","timeip-10-150-1-55.us-west-2.compute.internal","msg":""pid":4759[1] Transition from WaitActive to WaitQuorum"}
"{:""msg2023-09-22T23:14:58.68942453Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal,"[1] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) Active WaitActive New ds_transition to WaitQuorum,"","v":0,"name":"crucible","pid"level:":475930}
{""v"msg"::"0,"time",":name"":"[0] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) WaitActive New New ds_transition to WaitQuorum2023-09-22T23:14:58.689461646Z"crucible,"hostname":"ip-10-150-1-55.us-west-2.compute.internal"","pid",:4759}"
{""msg":"level":30[1] Transition from WaitActive to WaitQuorum","v":,0,""vname":"crucible","level":30{":0,"name":""crucible"msg,",:"""timetime"":":Crucible stats registered with UUID: c72ddd63-a86d-4aaf-9e6d-66c5f8903115"2023-09-22T23:14:58.689502575Z"",",v"":2023-09-22T23:14:58.689509924Z0,"","hostname":","level":ip-10-150-1-55.us-west-2.compute.internal"30,"pid":name4759":"}
,{"time":"2023-09-22T23:14:58.689542405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
hostname{":""msg":"ip-10-150-1-55.us-west-2.compute.internal","[0] Transition from WaitActive to WaitQuorumpid"":,"4759v":0,"}name
""msg:"":"crucible","level":30{[1] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) Active WaitQuorum New ds_transition to Active",""v":msg0","crucible","nametime"",::"crucible"","levellevel":":3030"2023-09-22T23:14:58.689596654Z","hostname":":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,}"
time[1] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) Active WaitQuorum New ds_transition to Active"":"{,"2023-09-22T23:14:58.689623323Z""msg":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid[0] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) WaitQuorum New New ds_transition to Active"":4759,"v"}:
,{"msg":"[1] Transition from WaitQuorum to Active","v":v0","name:":"0crucible",",level":"30name":"crucible","level":30,"time":"2023-09-22T23:14:58.689673451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"time":""msg":"2023-09-22T23:14:58.68962436Z","time":","hostname":2023-09-22T23:14:58.689678129Z""[2] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) Active Active New ds_transition to WaitActive",","hostnameip-10-150-1-55.us-west-2.compute.internal"v,"0"pid":,"4759name:":0","crucible"name}"
:"crucible","level":30":",{"level":ip-10-150-1-55.us-west-2.compute.internal"30,"pid":"4759msg":"}
,","time"time:"Crucible c72ddd63-a86d-4aaf-9e6d-66c5f8903115 has session id: 63f8da4b-43a4-40e6-97e2-4347bb217a59":{,"""v":2023-09-22T23:14:58.689726088Z2023-09-22T23:14:58.689721035Z0",","msg"hostname:""":name":"[1] Transition from WaitQuorum to Activecrucible""",,"vip-10-150-1-55.us-west-2.compute.internal"",""pid":4759}
:0,"name":"crucible","level":30{"msg":"[2] Transition from New to WaitActive","v":0,"name":""crucible","level":,30"hostname":",levelip-10-150-1-55.us-west-2.compute.internal""",":time"30pid":":4759,2023-09-22T23:14:58.689789317Z"}time
"{:"2023-09-22T23:14:58.68980531Z""msg",:""hostname":"[0] Transition from WaitQuorum to Active",ip-10-150-1-55.us-west-2.compute.internal"","hostname":",ip-10-150-1-55.us-west-2.compute.internal"","pid":time"4759:"}
2023-09-22T23:14:58.689816448Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid""msg":":4759}
,[2] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) Active Active New ds_transition to WaitActive""pid,""v{"::04759,"name":""crucible"msg",:""level":30[0] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.689875598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
}
,"time":"2023-09-22T23:14:58.689886372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"{v":0,"name":""msg":crucible{""msg":"{[2] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) Active Active WaitActive ds_transition to WaitQuorum","v":"0msg":","name":"[2] Transition from New to WaitActive"crucible",","level":30v":0,"name":"crucible","level":30,"time":"[0] Transition from New to WaitActive"2023-09-22T23:14:58.689948424Z",,""v":hostname":"0,"name"ip-10-150-1-55.us-west-2.compute.internal","time":"2023-09-22T23:14:58.689956941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"pid":4759}
,"time":"2023-09-22T23:14:58.689997855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30{"msg":"[2] Transition from WaitActive to WaitQuorum","v":,0","time":"name":"2023-09-22T23:14:58.690024341Z"crucible",,":,""level":30crucible","level":30,"time":""2023-09-22T23:14:58.690049087Z"level":,"30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759},"
,time"":time{":"msg":""hostname"[1] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) Active New New ds_transition to WaitActive:"","v"ip-10-150-1-55.us-west-2.compute.internal:"0,,""pid":name4759":"}
crucible","level":{30"msg":""2023-09-22T23:14:58.690067915Z",[2] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) Active Active WaitQuorum ds_transition to Active"",hostname""v",::"0ip-10-150-1-55.us-west-2.compute.internal"","pid":47592023-09-22T23:14:58.690053453Z"}
{"msg":","[2] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) Active Active WaitQuorum ds_transition to Active"hostname,,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.69015452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"v":0,"name":"{crucible","level"":msg30":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30",","timetime""::"2023-09-22T23:14:58.690183743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"{:""msg":"ip-10-150-1-55.us-west-2.compute.internal"[2] Transition from WaitQuorum to Active",","pid"v:":04759,"name":"}crucible""
2023-09-22T23:14:58.690110235Z,""level":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal{","pid":4759}"
,"time":"msg":"2023-09-22T23:14:58.690236963Z"{,"hostname":"time""ip-10-150-1-55.us-west-2.compute.internal:"[0] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) WaitActive New New ds_transition to WaitQuorum""2023-09-22T23:14:58.690189735Z,"v":0,""name":"crucible",","hostname"level"::"30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"1c9731f3-b505-4ad4-b45b-f22ac14fff3b is now active with session: 51812711-0ce3-446b-bb9f-9ece34c97797","v":0,"name":"crucible","level,":"30pid,":msg,"":time":""2023-09-22T23:14:58.69028777Z","hostname":[1] Transition from New to WaitActive""ip-10-150-1-55.us-west-2.compute.internal",",pid":"4759v":}
04759,"name"}:"
"{"msg":"a5c9346e-78da-47f3-a9ba-6cf4321f87bd is now active with session: be7cf715-24aa-4419-8cc0-e5ddcc58d64d","v":0,"name":"crucible","level":30time":"2023-09-22T23:14:58.690267713Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759crucible[1] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) Active Active Active ds_transition to Faulted}",,
""vtime":"2023-09-22T23:14:58.690366257Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
":0,"name":"crucible","level":30",,""{leveltime""":msg30"msg":"[0] Transition from WaitActive to WaitQuorum","v":,0","timename":"":crucible"","2023-09-22T23:14:58.690435121Zlevel""::"[1] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30:"2023-09-22T23:14:58.690418167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.690473068Z",""hostname"msg":":"[1] Transition from Active to Faultedip-10-150-1-55.us-west-2.compute.internal30"",,""v"pid:"0:,"4759name":"crucible"},"
,{""msg":","[1] Transition from Active to Faulted"hostname,"":"v":0,"name"ip-10-150-1-55.us-west-2.compute.internal:"","crucible"pid,":4759}
level":30{"msg":"[1] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) Active WaitActive New ds_transition to WaitQuorum","v":0,",time":""name":"2023-09-22T23:14:58.690561714Z""level,"":hostname"30:"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"time":","{2023-09-22T23:14:58.690496112Ztime":"""cruciblemsg,":2023-09-22T23:14:58.690583951Z"",hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759"}
,"level":30{"msg":""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":[0] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) WaitQuorum New New ds_transition to Active4759,"}"
"{"[1] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) Active Faulted Active ds_transition to LiveRepairReady"msg":","v":0,"name":"crucible","level":30[1] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) Active Faulted Active ds_transition to LiveRepairReady","v":0,"timename":"":"crucible",,2023-09-22T23:14:58.690623768Z""level":,,"time":""2023-09-22T23:14:58.690693114Z"hostname,"":hostname":""ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"}pid"
:4759}{
"msg":"[1] Transition from Faulted to LiveRepairReady{30"","msgv":":"0,"name":"[1] Transition from WaitActive to WaitQuorumcrucible""",,""level"v"v":::,"0time",:"302023-09-22T23:14:58.690729159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady",",v"":time"0:,"""name"name:""2023-09-22T23:14:58.690748971Z"crucible",,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":47590}
,"name":{"crucible"",msg":""level":30[1] 1c9731f3-b505-4ad4-b45b-f22ac14fff3b (51812711-0ce3-446b-bb9f-9ece34c97797) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30level":30:"crucible","level":30,"time":"2023-09-22T23:14:58.690807432Z",,"",hostname""time":","time":"2023-09-22T23:14:58.690807696Z","2023-09-22T23:14:58.690815795Z"hostname":,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"ip-10-150-1-55.us-west-2.compute.internal:"4759,"pid"}:
{"msg":"4759:"}
[1] a5c9346e-78da-47f3-a9ba-6cf4321f87bd (be7cf715-24aa-4419-8cc0-e5ddcc58d64d) Active LiveRepairReady Active ds_transition to LiveRepair"{"msg":"time[1] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) Active WaitQuorum New ds_transition to Active"",:""v":0,"2023-09-22T23:14:58.690800617Zname":""crucible","level":,30"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759},"
,v"":time"0:","name":"2023-09-22T23:14:58.690875331Z"crucible",,""hostnamelevel""{::"30ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",,""msg"pid"::"4759","pidtime"[0] Transition from WaitQuorum to Active"":","v":2023-09-22T23:14:58.690895389Z"0,,""name":"crucible","}level
":30{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30hostname":","timeip-10-150-1-55.us-west-2.compute.internal"":","pid":,"2023-09-22T23:14:58.690920841Z"4759time"},
:{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name4759":"crucible"},
"level":30{"msg","time"":"hostname"2023-09-22T23:14:58.690992288Z":","hostname":"ip-10-150-1-55.us-west-2.compute.internal",ip-10-150-1-55.us-west-2.compute.internal""pid",:"pid"4759:4759}
{}
{"msg""msg":"[1] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) Active New New ds_transition to WaitActive","v"::"0,"name":"[1] Transition from WaitQuorum to Active"crucible",",level"":v":300,"name":"crucible","level":30:"Write to Extent 0:2:9 under repair","v":0,"name":"crucible","level":40,,""time":time",""time":":2023-09-22T23:14:58.691069933Z"2023-09-22T23:14:58.691082007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
:"2023-09-22T23:14:58.690929802Z","hostname":"2023-09-22T23:14:58.69107416Z"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"}
hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":"Write 1:0:9 past extent under repair 0","v":0,"name":"crucible","level":40,"""msgtime""::""2023-09-22T23:14:58.691175336Z",","hostnamehostname"[2] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) Active Active New ds_transition to WaitActive":":"","ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"pid":4759"},"
pid":4759}
{{""msg"msg":":"[1] Transition from New to WaitActive"Write 1:1:9 past extent under repair 0",",v""v"::0,0","name":"crucible","namelevel"":":crucible"30,"level":40v":0,"name":"crucible",","level"time","time":"2023-09-22T23:14:58.691245693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
:"2023-09-22T23:14:58.691246774Z","hostname"{:":ip-10-150-1-55.us-west-2.compute.internal","pid"30":4759msg":}"
{[1] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) Active WaitActive New ds_transition to WaitQuorum"","msg"v":":0IO Write 1008 on eur 0 Added deps 1",,""v"name":":0crucible,""name,,""level:"":crucible","level":40,"time":"2023-09-22T23:14:58.691321989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
30"time":"2023-09-22T23:14:58.691294406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.691352592Z"{,"hostname":""ip-10-150-1-55.us-west-2.compute.internal"msg":","pid":4759[2] Transition from New to WaitActive","}v
":0,"name":"crucible","{level":30"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30test live_repair::repair_test::test_live_repair_deps_repair_rspan_right ... ,ok"
time":"2023-09-22T23:14:58.691399517Z",","hostnametime":":""2023-09-22T23:14:58.69141105Z"ip-10-150-1-55.us-west-2.compute.internal",,""hostname"pid":":4759ip-10-150-1-55.us-west-2.compute.internal}"
,"pid":4759}
{"{msg":""msg":"[2] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) Active Active WaitActive ds_transition to WaitQuorum","v":0,[1] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) Active WaitQuorum New ds_transition to Active""name":,""v"crucible":,"0level":,"30name":"crucible","level":30,"time":"2023-09-22T23:14:58.691473716Z",,""hostname"time":":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.69147679Z"",",pid"":hostname4759":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759{}
"msg":"[2] Transition from WaitActive to WaitQuorum"{,"v":0,"name"":"msg":crucible"","level":30[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.691530115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"time"}:
"2023-09-22T23:14:58.691536694Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[2] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) Active Active WaitQuorum ds_transition to Active","v":0,"{name":"crucible","level":"30msg":"[2] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) Active Active New ds_transition to WaitActive","test live_repair::repair_test::test_live_repair_deps_repair_wafter ... v",ok:
"0time":",2023-09-22T23:14:58.691581094Z"","namehostname":":""crucible"ip-10-150-1-55.us-west-2.compute.internal",,""pid":level":475930}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,,""time":"time":"2023-09-22T23:14:58.691628255Z","hostname":"2023-09-22T23:14:58.691617832Z"ip-10-150-1-55.us-west-2.compute.internal",,""pidhostname""::"4759}
ip-10-150-1-55.us-west-2.compute.internal","{pid":"4759msg":"}
b28e974a-c998-41d2-b8a0-fef8e732ae88 is now active with session: 8884c3fe-403a-4534-b3a2-b199a5a05d84","v":0,"name{":"crucible","level":"30msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.691665948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"{time":""2023-09-22T23:14:58.691674412Z"msg":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) Active Active Active ds_transition to Faulted",}"
v":0,"name":"crucible"{,"level":30"msg":"[2] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.69171284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.691726256Z",""hostnamemsg":"":"[1] Transition from Active to Faulted"ip-10-150-1-55.us-west-2.compute.internal,"","v"pid"::04759,"name":}"
crucible","level":30{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.691767922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.691779927Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) Active Faulted Active ds_transition to LiveRepairReady","v":0,"{name":"crucible",""levelmsg":"":30[2] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible"{,"level":30"msg,""time":":"2023-09-22T23:14:58.691826985Z","hostname":"Crucible stats registered with UUID: 4755c4b6-38b2-4fc5-b920-a3b417f63d0d",ip-10-150-1-55.us-west-2.compute.internal"",v"","pid"::04759time,""name:""}:"
2023-09-22T23:14:58.691837888Z"crucible",,""{hostname":"level""msgip-10-150-1-55.us-west-2.compute.internal""::"30,"pid":4759[1] Transition from Faulted to LiveRepairReady",}"
v":0,"name":"crucible",{"level":30"msg":"[2] Transition from WaitQuorum to Active","v":0,",name"":time":""crucible"2023-09-22T23:14:58.691880887Z,"",time":"","2023-09-22T23:14:58.691895244Z"hostname",:""level"hostname":ip-10-150-1-55.us-west-2.compute.internal"":,30ip-10-150-1-55.us-west-2.compute.internal"",pid"":pid"4759:4759}}

{,"time":"{"msg2023-09-22T23:14:58.691922917Z":"""msg",:""hostname":"Crucible 4755c4b6-38b2-4fc5-b920-a3b417f63d0d has session id: 89f34d5a-f22c-4d68-bcdd-5b31cf1d61b8","ip-10-150-1-55.us-west-2.compute.internal"v,""[1] b28e974a-c998-41d2-b8a0-fef8e732ae88 (8884c3fe-403a-4534-b3a2-b199a5a05d84) Active LiveRepairReady Active ds_transition to LiveRepair":,0pid":"4759,v"}:
0","name"name":":{"crucible"crucible,""level",":"30level"msg"::"30c72ddd63-a86d-4aaf-9e6d-66c5f8903115 is now active with session: d9db3e91-27d7-4cb9-af2b-1cf3b9294be1","v":0,"name":"crucible","level":30,"time":",2023-09-22T23:14:58.69197879Z"","time":"hostname":"2023-09-22T23:14:58.691982245Z"ip-10-150-1-55.us-west-2.compute.internal,"","hostname,pid"":time4759":"}:"
"2023-09-22T23:14:58.691992116Z","ip-10-150-1-55.us-west-2.compute.internal"hostname,"{pid""::4759""msg"}:
"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] Transition from LiveRepairReady to LiveRepair"{},
""v"msg:":0{","name":"crucible"",msg":""[0] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) New New New ds_transition to WaitActive"level":,"30v":0[1] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) Active Active Active ds_transition to Faulted",",name":""cruciblev"":,0",level"":name":30"crucible",","time"level"::"302023-09-22T23:14:58.692060952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.692074326Z","",hostname":""msg"time":ip-10-150-1-55.us-west-2.compute.internal"":,"2023-09-22T23:14:58.69208152Zpid""":,"4759Write to Extent 0:2:9 under repair"hostname},"
":v":"0,"{name"ip-10-150-1-55.us-west-2.compute.internal":,"""pid"msg"::crucible4759"",}"
level"[0] Transition from New to WaitActive":40,"v"{:0,"name":""crucible"msg",:""level":30[1] Transition from Active to Faulted","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:14:58.69214181Z"{,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","",pid"msg"::4759"}
Crucible stats registered with UUID: 3bb7876f-b8c5-49a9-9144-66268618f386","time,":""v":2023-09-22T23:14:58.692158374Z"0,,""hostname":name":""crucibletime":"2023-09-22T23:14:58.692150972Z","hostname":"","levelip-10-150-1-55.us-west-2.compute.internal"":,"30pid":4759}
{""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[0] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) WaitActive New New ds_transition to WaitQuorum"},,""v"time:":0","name":2023-09-22T23:14:58.692196077Z""crucible",","hostname"level:"":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.692224723Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
Crucible 3bb7876f-b8c5-49a9-9144-66268618f386 has session id: 80597233-0197-410e-81fc-216b4e403404{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.692264048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",""pid":4759,"v"}:
0,"name":"crucible{","level":"30msg":"
[0] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible",",time"{level""::"302023-09-22T23:14:58.692291638Z",""hostname":msg"","time":"2023-09-22T23:14:58.692306364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":""msg":"2023-09-22T23:14:58.692337929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","[0] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) New New New ds_transition to WaitActivepid"":,4759"v":}
0:,{""name"":msg"":"crucible","level":30[1] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) Active Faulted Active ds_transition to LiveRepairReady","v":0[1] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) Active New New ds_transition to WaitActive",",name"":v":"0crucible,",,"level"":30time":"2023-09-22T23:14:58.692364834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"name":""msgcrucible"":",,""[0] Transition from New to WaitActivetimelevel""",:"2023-09-22T23:14:58.692380017Z"test live_repair::repair_test::test_live_repair_deps_repair_write ... ,"okhostname
":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"v"":msg":"0,"name":"[1] Transition from Faulted to LiveRepairReady"crucible",",v":"0level",":name":"30crucible","level":30,"time":"2023-09-22T23:14:58.692443999Z","hostname":,":"time30":"ip-10-150-1-55.us-west-2.compute.internal",2023-09-22T23:14:58.69244648Z"","pid"hostname":,"time":"2023-09-22T23:14:58.692464536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"4759[1] Transition from New to WaitActive"}
{"msg":"[0] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) WaitActive New New ds_transition to WaitQuorum",,""v":v"0:,"0name":,""namecrucible"":",:crucible""ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] c72ddd63-a86d-4aaf-9e6d-66c5f8903115 (d9db3e91-27d7-4cb9-af2b-1cf3b9294be1) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30level":30,"time":"2023-09-22T23:14:58.69253704Z"","hostname":","level,""time:":"2023-09-22T23:14:58.692543734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":3030,"time":"2023-09-22T23:14:58.69258263Z","hostname":","time":ip-10-150-1-55.us-west-2.compute.internal"","pid":47592023-09-22T23:14:58.692588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) Active WaitActive New ds_transition to WaitQuorum","v":0,}"
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":{30"msg":"[0] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) WaitQuorum New New ds_transition to Active","v":0,"name":"namecrucible"",:""level":crucible"30,","time"level:"":302023-09-22T23:14:58.692658656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time,"":"time":"2023-09-22T23:14:58.692671444Z"2023-09-22T23:14:58.692675683Z,"",hostname"":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internal"pid",:"4759pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0},"name
":"crucible","level"{:30"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.692714509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg",:""time":"2023-09-22T23:14:58.692724301Z","hostname":"[1] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) Active New New ds_transition to WaitActive","v"ip-10-150-1-55.us-west-2.compute.internal:"0,","pid":name":4759"crucible"}
,"level":30{"msg":"[1] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible",","level"time:":30"2023-09-22T23:14:58.692762904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.692775806Z"",msg"":"hostname":"[1] Transition from New to WaitActive",ip-10-150-1-55.us-west-2.compute.internal""v,"":pid0":,"4759name":}"
crucible","level"{:30"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.692798433Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"time}"
{"msg":"[1] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) Active WaitActive New ds_transition to WaitQuorum":,"v"":0,"name"2023-09-22T23:14:58.692806853Z:""crucible",,""levelhostname""::"30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{,""time"msg:"":"2023-09-22T23:14:58.692835738Z",""hostnamemsg"":"[2] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) Active Active New ds_transition to WaitActive:"","v":0,ip-10-150-1-55.us-west-2.compute.internal"","pid":4759}
name{":""crucible"msg":","level":30[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30Crucible stats registered with UUID: af9c4a58-0d51-4892-b110-0ff415b3afef","v":0,"name":"crucible",",time",time""level":30:"2023-09-22T23:14:58.692879492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,""time"msg:"":"test live_repair::repair_test::test_live_repair_deps_repair_wspan_left ... 2023-09-22T23:14:58.692893017Z",ok"hostname
":"[1] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) Active WaitQuorum New ds_transition to Activeip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible af9c4a58-0d51-4892-b110-0ff415b3afef has session id: 5d96c4af-b83f-49c3-b296-a00e3993f867"","v":0,,""name"v:"":crucible0",","namelevel":"30:"crucible","level":30":"2023-09-22T23:14:58.692874543Z",","time":hostname":"","timeip-10-150-1-55.us-west-2.compute.internal"",:"2023-09-22T23:14:58.692968105Z","hostname":""pid":4759ip-10-150-1-55.us-west-2.compute.internal","}pid
":4759}
{2023-09-22T23:14:58.69296285Z{"""msg"msg:"":","hostname[2] Transition from New to WaitActive[1] Transition from WaitQuorum to Active""",":"vip-10-150-1-55.us-west-2.compute.internal",""pid"::47590,"name}"
:"crucible","level":{30",msg"":"v":0,"name":"crucible","level"[0] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) New New New ds_transition to WaitActive",,"v":0,""name"time":"2023-09-22T23:14:58.69302446Z":,"30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
:"{crucible","",msg""level:"":30[2] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30time":"2023-09-22T23:14:58.693047994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759},
","time":"time":{2023-09-22T23:14:58.693074672Z"",""msghostname""2023-09-22T23:14:58.693069119Z":,"hostname":""ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid"::4759}4759
}
{{""msg"msg":":"[2] Transition from New to WaitActive","v"[2] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) Active Active WaitActive ds_transition to WaitQuorum:"0,",name"":v"":0crucible",","name":level"":crucible30","level":30:"[0] Transition from New to WaitActive","v":0,",",name"time"::""crucible","2023-09-22T23:14:58.69313506Zlevel":"30,"hostname":""timeip-10-150-1-55.us-west-2.compute.internal"":,""pid":47592023-09-22T23:14:58.693136881Z"}
,"time":"{2023-09-22T23:14:58.693157683Z"","msg":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[2] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) Active Active WaitActive ds_transition to WaitQuorum}"
,"v":0,"name":,"{crucible"msg":"","level":30[0] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.693202225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{""msg"hostname":":"[2] Transition from WaitActive to WaitQuorum"ip-10-150-1-55.us-west-2.compute.internal","v,"":pid"0:,"4759name":"}
crucible{"",msg"":level"":30[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.693211625Z",","hostname"time"::,"time"":"2023-09-22T23:14:58.69325047Z"2023-09-22T23:14:58.693257263Z",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{"msg":""ip-10-150-1-55.us-west-2.compute.internal"[2] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) Active Active WaitQuorum ds_transition to Active,{"",pid"""msgv":"":0,"name":"crucible","level":30[2] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) Active Active WaitQuorum ds_transition to Active","v":0,"name"::"crucible"4759,"level":}30
,"time":"2023-09-22T23:14:58.693307412Z","{hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal"",msg"":pid":4759}
{"msg":"2023-09-22T23:14:58.693315903Z"[2] Transition from WaitQuorum to Active",,""v":hostname"0:","name":""ip-10-150-1-55.us-west-2.compute.internalcrucible"",","pid":level":475930[0] Transition from WaitActive to WaitQuorum"}
,"v":0,{"name":""crucible"msg":,","level":30time":"2023-09-22T23:14:58.693350915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":","time":"""2023-09-22T23:14:58.69336383Z"3bb7876f-b8c5-49a9-9144-66268618f386 is now active with session: 910334ff-5f0d-4dec-a302-b396ca5c2b47"[2] Transition from WaitQuorum to Activemsg",","":"vv""::00Crucible stats registered with UUID: 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed,",""v"name:":"0crucible",",,"name""levelhostname":":crucible"30,"level":30,"time":"2023-09-22T23:14:58.693413505Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) Active Active Active ds_transition to Faulted","v",:"0time"":":,""2023-09-22T23:14:58.693416317Zname""ip-10-150-1-55.us-west-2.compute.internal":,"pid":4759},"
crucible","level"{:30,""msg":"hostname":""name":ip-10-150-1-55.us-west-2.compute.internal""crucible","[0] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) WaitQuorum New New ds_transition to Active",,"",time"":level":"30v"2023-09-22T23:14:58.693465122Z:"0,","hostname"name":pid":"4759,ip-10-150-1-55.us-west-2.compute.internal"",time"":}pid
":"crucible","level":{30"msg":"Crucible 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed has session id: 55205457-cbd1-4d6f-81eb-71ee3333eada","v":0,"name":"crucible","level":30:4759,"}time"
"{2023-09-22T23:14:58.693483309Z""msg,"":"hostname":"[1] Transition from Active to Faulted",:""ip-10-150-1-55.us-west-2.compute.internal,"2023-09-22T23:14:58.693509786Z"v":,0time","hostname":""name":"crucible"ip-10-150-1-55.us-west-2.compute.internal,"level":30"",",pid"":pid":47594759}}

,{"{Job is DownstairsIO { ds_id: JobId(1002), guest_id: 1, work: ExtentFlushClose { dependencies: [JobId(1000), JobId(1001)], extent: 1, flush_number: 2, gen_number: 4, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 1, block: 0 }, ImpactedAddr { extent_id: 1, block: 2 }) }
:time"2023-09-22T23:14:58.693524992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":"4759msg"}:
"{""msg"msg":":"[0] Transition from WaitQuorum to Active","v":0,"[0] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) New New New ds_transition to WaitActivename":""crucible","level":,"30v"::0","name":"2023-09-22T23:14:58.69355967Zcrucible"","level":,30"hostname":""ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
4755c4b6-38b2-4fc5-b920-a3b417f63d0d is now active with session: 29aa10dc-2c76-4f2c-8851-5916dfa1f48d","time":","{v":2023-09-22T23:14:58.693705021Z"0","msg,"":name""hostname":",":"time"[1] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) Active Faulted Active ds_transition to LiveRepairReadycrucible"",ip-10-150-1-55.us-west-2.compute.internal":","",2023-09-22T23:14:58.693713361Zpid"v"level:"0"::,"304759name"}":
","crucible"hostname","{level":":"30msgip-10-150-1-55.us-west-2.compute.internal"":",","timepid":"[1] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) Active New New ds_transition to WaitActive":,"4759",2023-09-22T23:14:58.69375028Z""time}v":
0,"":"hostname","2023-09-22T23:14:58.693765185Z:"",{"namehostnameip-10-150-1-55.us-west-2.compute.internal"":,""""pid"msg:":ip-10-150-1-55.us-west-2.compute.internal"4759,"}pid
"crucible":""{:,""level":msg4759":"}30
[0] Transition from New to WaitActive",[1] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) Active Active Active ds_transition to Faulted{"",""v",":timevmsg""::0","":name[1] Transition from Faulted to LiveRepairReady"":",crucible""v,"":"0level,"":name30"2023-09-22T23:14:58.693818056Z"0:","cruciblehostname","":name":,,"""timelevel"":"ip-10-150-1-55.us-west-2.compute.internal"":,"crucible"2023-09-22T23:14:58.693844996Z30,"pid,""level"hostname":"4759:,:""time}":ip-10-150-1-55.us-west-2.compute.internal""
2023-09-22T23:14:58.693863963Z,""{30pid,"":"4759hostname"}:
"msg":"{[1] Transition from New to WaitActive"ip-10-150-1-55.us-west-2.compute.internal,""",v":msg""pid:"":04759,"[1] Transition from Active to Faulted}"
,name":{,"""msgv"":"time":0","crucible"name":",":cruciblelevel":30""[1] 3bb7876f-b8c5-49a9-9144-66268618f386 (910334ff-5f0d-4dec-a302-b396ca5c2b47) Active LiveRepairReady Active ds_transition to LiveRepair,"",level""v:"30:2023-09-22T23:14:58.693884855Z"0,",,"name"time":"":"2023-09-22T23:14:58.693921071Z"hostname,"crucible,":""hostname",""timelevel""::"30ip-10-150-1-55.us-west-2.compute.internal":"2023-09-22T23:14:58.693925442Z",,""ip-10-150-1-55.us-west-2.compute.internal"pid","hostname,"pid"::""time"::47594759"ip-10-150-1-55.us-west-2.compute.internal"}2023-09-22T23:14:58.693946584Z,"",pid"":
4759hostname"}:
"}{
{ip-10-150-1-55.us-west-2.compute.internal",{"""pid"":msg4759msg":"msg"}"
:"[1] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) Active WaitActive New ds_transition to WaitQuorum"{,":v":0[1] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) Active Faulted Active ds_transition to LiveRepairReady,"",msg"""":v""name":":[1] Transition from LiveRepairReady to LiveRepair0",",name""v:"":crucible0crucible,"","name"":level""[0] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) WaitActive New New ds_transition to WaitQuorum","crucible:"30,level":"30,level":"30v":0,"name":",","crucible"time",time,"":"timelevel"""::"2023-09-22T23:14:58.694022685Z""2023-09-22T23:14:58.694025353Z",2023-09-22T23:14:58.694028441Z"":,hostname"",hostname:"":"30"hostnameip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"",pid""":":pid"4759:4759ip-10-150-1-55.us-west-2.compute.internal"}}

,"pid":4759}{
"msg":"{"msg":",[1] Transition from Faulted to LiveRepairReady"[1] Transition from WaitActive to WaitQuorum","","time":"v":v"2023-09-22T23:14:58.694054778Z"0:,,"0hostname":",""name"ip-10-150-1-55.us-west-2.compute.internal"name:"",crucible:""crucible""pid",":,"level":4759level":3030}
,"time":"2023-09-22T23:14:58.694105568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759}
",msg"":time{"":""msg":"[0] Transition from WaitActive to WaitQuorum"2023-09-22T23:14:58.694106381Z","v":,0"[1] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) Active WaitQuorum New ds_transition to Activehostname"":,""name":",crucible"ip-10-150-1-55.us-west-2.compute.internal""v,,""level":"0pid",":name4759"::}30"
crucible","level":30,"time":"2023-09-22T23:14:58.694159252Z","hostname":","ip-10-150-1-55.us-west-2.compute.internaltime":"","pid":2023-09-22T23:14:58.694169329Z"4759,"hostname"}:
"ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"{msg":"{"msg":""msg[0] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) WaitQuorum New New ds_transition to Active"":","v":0[1] Transition from WaitQuorum to Active","[1] 4755c4b6-38b2-4fc5-b920-a3b417f63d0d (29aa10dc-2c76-4f2c-8851-5916dfa1f48d) Active LiveRepairReady Active ds_transition to LiveRepair"name",,"":"vv":":00,"crucible"name,":,""cruciblelevel":"30","name"level:"":crucible"30,"level":30,"time":","2023-09-22T23:14:58.694239838Z"time":",",2023-09-22T23:14:58.694244594Z""hostname",time"::"""hostname":"2023-09-22T23:14:58.6942469Z"ip-10-150-1-55.us-west-2.compute.internal",ip-10-150-1-55.us-west-2.compute.internal"",,""pid"hostname"::4759pid"":}4759
}ip-10-150-1-55.us-west-2.compute.internal"
,{"pid":{4759""}msgmsg"
:"":"[0] Transition from WaitQuorum to Active"{,"v"[1] Transition from LiveRepairReady to LiveRepair""msg":,":0"v":,"0name,":"test live_repair::repair_test::test_live_repair_deps_super_spanner ... name"ok:"
crucible"[2] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) Active Active New ds_transition to WaitActive"",",cruciblev":"0level,"""name,""level":::3030"crucible","level":30,,""time":time":""2023-09-22T23:14:58.694355231Z",,"2023-09-22T23:14:58.694352756Z""hostname,time":"":""2023-09-22T23:14:58.694353451Z"hostname":ip-10-150-1-55.us-west-2.compute.internal",""hostname",:ip-10-150-1-55.us-west-2.compute.internal""",pid"ip-10-150-1-55.us-west-2.compute.internal""pid",:":4759pid"4759}:}4759
}

{"msg":{"[2] Transition from New to WaitActive",""msg"v":":0,"name":"crucible","level":30[1] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.694430191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"time":"}
2023-09-22T23:14:58.694437295Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal"",msg"":"pid":4759}
[2] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"{crucible","level":30"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.694481957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.694490924Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","[2] Transition from WaitActive to WaitQuorum"pid",:"4759v":0},
"name":"crucible","level":{30"msg":"[1] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level,"":time"30:"2023-09-22T23:14:58.694532488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,""timemsg":"":"2023-09-22T23:14:58.694548786Z","hostname":"[2] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) Active Active WaitQuorum ds_transition to Active"ip-10-150-1-55.us-west-2.compute.internal",,""pidv""::47590,"name":"}crucible
","level":30{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.69458955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"{time":""2023-09-22T23:14:58.694602911Z"msg":","hostname":"[2] Transition from WaitQuorum to Active","v":ip-10-150-1-55.us-west-2.compute.internal"0,",pid""name":":crucible4759","level"}:
30test live_repair::repair_test::test_live_repair_deps_repair_wspan_right ... {ok
","msg":time"":"2023-09-22T23:14:58.694640919Z","hostname":"[1] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) Active WaitQuorum New ds_transition to Active",ip-10-150-1-55.us-west-2.compute.internal"","v"pid:":04759,"name":"}
crucible","level":{30"msg":"af9c4a58-0d51-4892-b110-0ff415b3afef is now active with session: 90c473a8-fc29-4ea8-a43c-81300d27fad5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.694682492Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal":,""pid":2023-09-22T23:14:58.694693995Z"4759,"hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible"[1] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) Active Active Active ds_transition to Faulted",",level"":v30":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.694741088Z",","time"hostname":":"2023-09-22T23:14:58.694745545Zip-10-150-1-55.us-west-2.compute.internal"",","pid":hostname4759":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}{
"msg":"{"msg"[2] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) Active Active New ds_transition to WaitActive":","v":[1] Transition from Active to Faulted0",",name":""crucible"v,"":level"0:,"30name":"crucible","level":30{,"time":",""2023-09-22T23:14:58.694805036Zmsg"time":":""2023-09-22T23:14:58.694808073Z",",hostname":""Crucible stats registered with UUID: c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab"hostname":"ip-10-150-1-55.us-west-2.compute.internal",,ip-10-150-1-55.us-west-2.compute.internal"","pid""pid"::47594759v":}}
0
{,"name":"crucible"{,""level"msg""msg":::30""[2] Transition from New to WaitActive","v":0,"name":"[1] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) Active Faulted Active ds_transition to LiveRepairReady"crucible",,""level"v:":300,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.694867888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759",time":""}time
":2023-09-22T23:14:58.694880771Z"","{hostname"2023-09-22T23:14:58.694885375Z":"",msg"":"ip-10-150-1-55.us-west-2.compute.internal"hostname",:""pidCrucible c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab has session id: 62b5a2a7-dfe7-4c3b-a240-ba1a85e4e77f""ip-10-150-1-55.us-west-2.compute.internal,"":,v4759"":pid"0}:,4759"
}name"
:"crucible","level"{:30"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.694950615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid",":time4759":"}
2023-09-22T23:14:58.69496082Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759"msg":}"
[0] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) New New New ds_transition to WaitActive","v":{0,"name":"crucible""msg":","Job is DownstairsIO { ds_id: JobId(1002), guest_id: 1, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001)], extent: 1 }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 1, block: 0 }, ImpactedAddr { extent_id: 1, block: 2 }) }
{[1] af9c4a58-0d51-4892-b110-0ff415b3afef (90c473a8-fc29-4ea8-a43c-81300d27fad5) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible"","msglevel""::"30Crucible stats registered with UUID: 82297280-9384-4cb6-a5ef-2a47addce4c7","v":0,"name":"crucible","level":30{"msg":","time":"2023-09-22T23:14:58.695073169Z","hostname":[2] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) Active Active WaitActive ds_transition to WaitQuorum","v":"0,,"ip-10-150-1-55.us-west-2.compute.internaltime":"2023-09-22T23:14:58.695083195Z","level"hostname","::30"ip-10-150-1-55.us-west-2.compute.internal"","pid"name":":"4759pid"crucible}:"4759
{,"time":"2023-09-22T23:14:58.695113289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
Job is DownstairsIO { ds_id: JobId(1003), guest_id: 1, work: ExtentLiveNoOp { dependencies: [JobId(1001), JobId(1002)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 1, block: 0 }, ImpactedAddr { extent_id: 1, block: 2 }) }
"msg,":"Crucible 82297280-9384-4cb6-a5ef-2a47addce4c7 has session id: aad1ab20-eb49-4087-bb03-986281672803","v":0},"name"
:"crucible","level":30{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30"level":30,"time":"2023-09-22T23:14:58.695315537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.695325255Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":"4759msg,"}
time":"2023-09-22T23:14:58.695330627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
":"{{[0] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) New New New ds_transition to WaitActive"",msg":""v":"0[2] Transition from WaitActive to WaitQuorummsg","":,"name":""[0] Transition from New to WaitActive"v"crucible,"":,"0level"v":,:"30name":"0crucible",","name"level":":crucible"30,"level":,"30time":"2023-09-22T23:14:58.695404181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":"","msg":time":""2023-09-22T23:14:58.695412649Z"[0] Transition from New to WaitActive",",hostname"":"v"2023-09-22T23:14:58.695415897Z":ip-10-150-1-55.us-west-2.compute.internal,"","hostname0","pid":":name":"4759crucible"ip-10-150-1-55.us-west-2.compute.internal,""}level":
,30"pid":4759}{
","msgtime":"":{2023-09-22T23:14:58.695467951Z"","hostname":""msg"ip-10-150-1-55.us-west-2.compute.internal":,""[2] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) Active Active WaitQuorum ds_transition to Active"pid":4759,"}v
"[0] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) WaitActive New New ds_transition to WaitQuorum":{0,",""v"name":"msg":"crucible":,"0level,""[0] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) WaitActive New New ds_transition to WaitQuorum":name",:"30crucible""v":,0,""name":"level":crucible"30,"level":30,"time":"2023-09-22T23:14:58.695521901Z","hostname":","ip-10-150-1-55.us-west-2.compute.internaltime"":","pid":47592023-09-22T23:14:58.695514151Z",",time"}"
:"hostname":"{2023-09-22T23:14:58.695521314Z"ip-10-150-1-55.us-west-2.compute.internal",""hostname",msg":"":pid"[0] Transition from WaitActive to WaitQuorum"",:"4759ip-10-150-1-55.us-west-2.compute.internal"v"},
":pid"0:,4759"{name"}:"
crucible"","msg"level"{::30""msg":"[2] Transition from WaitQuorum to Active","v":[0] Transition from WaitActive to WaitQuorum0",",name":""cruciblev",:"0",",name":"time":""level"crucible2023-09-22T23:14:58.695583376Z:""30,test live_repair::repair_test::test_live_repair_deps_writes ... ,"okhostname"
:""level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.695609357Z",""hostname"msg,":":""time":"2023-09-22T23:14:58.695623867Zip-10-150-1-55.us-west-2.compute.internal"","[0] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) WaitQuorum New New ds_transition to Activepid",",""hostname":"v"::ip-10-150-1-55.us-west-2.compute.internal"04759,","name}":pid":
"4759crucible"},
{"level":30{"msg":""msg":"6d860aa7-cf0b-47de-b0a7-b9d3a68235ed is now active with session: 758265c4-df86-4cfd-92d1-c4c7741a69e6","v":[0] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) WaitQuorum New New ds_transition to Active"0,,""name":v"",crucible":,""0time,level":"""name"2023-09-22T23:14:58.6956737Z:":"30crucible",","hostname"level:"":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":",crucible"",,time"""timelevel":":":30"2023-09-22T23:14:58.6957073Z",2023-09-22T23:14:58.695702253Z""hostname":","hostname":"ip-10-150-1-55.us-west-2.compute.internal,""time",ip-10-150-1-55.us-west-2.compute.internal:"""2023-09-22T23:14:58.695729676Z",,""pidpid":"hostname:4759":"4759}ip-10-150-1-55.us-west-2.compute.internal"}
{
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30"msg":"[1] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) Active Active Active ds_transition to Faulted,"time":"2023-09-22T23:14:58.695829594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{""msg":","v":0,"name":"crucible",[1] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) Active New New ds_transition to WaitActive"","levelv":"0:,"30name":"crucible","level":30,"time":","2023-09-22T23:14:58.695880801Z"time":",",hostname""2023-09-22T23:14:58.695877633Z"pid"::,4759""}hostname
":"ip-10-150-1-55.us-west-2.compute.internal","pid":{4759ip-10-150-1-55.us-west-2.compute.internal"},
""pid":msg{4759":""}msg"
[1] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) Active New New ds_transition to WaitActive":","v":0[1] Transition from New to WaitActive,"",name":""{v":crucible"0,""level"msg:,30"":"name":"crucible"[1] Transition from Active to Faulted",",level":"30v":0,"name":","crucible"time",:""level"2023-09-22T23:14:58.695950915Z":,"30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{"2023-09-22T23:14:58.695959448Z"msg":","hostname":"[1] Transition from New to WaitActive","v":0,"ip-10-150-1-55.us-west-2.compute.internal"name",:""crucible"pid",",":level":30time"4759:"}
2023-09-22T23:14:58.695969761Z","time":","hostname2023-09-22T23:14:58.695992885Z"{,""hostname":""msg":ip-10-150-1-55.us-west-2.compute.internal":",""pid":4759}
[1] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) Active WaitActive New ds_transition to WaitQuorum"ip-10-150-1-55.us-west-2.compute.internal"{,","vmsg":"":"0pid",:"4759name":"[1] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) Active WaitActive New ds_transition to WaitQuorumcrucible"}"
,{",v":"0"level"msg,"{::"30name"":""crucible"msg":Crucible stats registered with UUID: 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6,""",level""v:":300,"name":"crucible","level":[1] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) Active Faulted Active ds_transition to LiveRepairReady30","v":0,"name":"crucible",,",""time"time":"level":2023-09-22T23:14:58.696055584Z"",","hostname"2023-09-22T23:14:58.696048676Z":time",":ip-10-150-1-55.us-west-2.compute.internal:"","302023-09-22T23:14:58.696060829Zpid"":,4759""hostname}"
:"hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid"ip-10-150-1-55.us-west-2.compute.internal"msg""::4759","}pid
[1] Transition from WaitActive to WaitQuorum"",:{,"""vmsg""::4759"0time"},
:"Crucible 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 has session id: e9bc9f9b-c734-4b03-90fb-6c8b80f7d91c""name",:"2023-09-22T23:14:58.696086447Z"{v"":,""0crucible,""msgname"",:""level"crucible:"30,":level":"30hostname":"[1] Transition from WaitActive to WaitQuorum","v"ip-10-150-1-55.us-west-2.compute.internal":,0","pid"name",,"":timetime""::"":crucible"2023-09-22T23:14:58.696137453Z""4759,2023-09-22T23:14:58.696141991Z"},
""level":,30"hostname"{:"ip-10-150-1-55.us-west-2.compute.internal",""pid":msg4759":"}
[1] Transition from Faulted to LiveRepairReady","v"{,":time"0:,""hostnamemsg""::""name":""ip-10-150-1-55.us-west-2.compute.internal"crucible,2023-09-22T23:14:58.696171883Z""[0] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) New New New ds_transition to WaitActive""pid",:,,"4759"}v
"level":"{:hostname""0msg,"":"name"::""30crucible"[1] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) Active WaitQuorum New ds_transition to Active",",level""v:"30:Job is DownstairsIO { ds_id: JobId(1003), guest_id: 1, work: ExtentLiveRepair { dependencies: [JobId(1001), JobId(1002)], extent: 1, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 1, block: 0 }, ImpactedAddr { extent_id: 1, block: 2 }) }
,",time":"2023-09-22T23:14:58.696222159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":"4759time0":},""
name2023-09-22T23:14:58.696226577Z"":"crucible",","hostnameip-10-150-1-55.us-west-2.compute.internallevel"{""::","30pid":"ip-10-150-1-55.us-west-2.compute.internalmsg""4759:",}"
pid"[1] 6d860aa7-cf0b-47de-b0a7-b9d3a68235ed (758265c4-df86-4cfd-92d1-c4c7741a69e6) Active LiveRepairReady Active ds_transition to LiveRepair":,"4759,time""}{v"
::"0msg","{:""2023-09-22T23:14:58.696376677Z""name":,msg""hostname:"":"[1] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) Active WaitQuorum New ds_transition to Active[0] Transition from New to WaitActive"""crucible"ip-10-150-1-55.us-west-2.compute.internal,"",v""pid:"0:,4759",}"
,v"name{":"":msgcrucible0"",:""level",":name[1] Transition from WaitQuorum to Active30""":"level",:crucible"30","v"level"::300,","time"name:"":"crucible",2023-09-22T23:14:58.696441599Z""level,"":30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time"":msg"",",2023-09-22T23:14:58.69645717Z:"time"time""",::"""2023-09-22T23:14:58.696452852Z"hostname":[0] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) WaitActive New New ds_transition to WaitQuorum"",",2023-09-22T23:14:58.696448653Z"hostname":,ip-10-150-1-55.us-west-2.compute.internal""v,"":""hostname":0pid,":"4759name"}:
"ip-10-150-1-55.us-west-2.compute.internal"{,""cruciblemsg"":,"""level"pid"::475930[2] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) Active Active New ds_transition to WaitActiveip-10-150-1-55.us-west-2.compute.internal""},
,""{v":",msg0",time""name:"":"":"crucible2023-09-22T23:14:58.696507891Z"",,""[1] Transition from WaitQuorum to Active"hostnamelevel""::"30pid",":v":04759,"ip-10-150-1-55.us-west-2.compute.internal"},name":"
crucible"",pid""time:"4759:,}"
2023-09-22T23:14:58.696532891Z"",{"levelhostname""{:msg"""::""ip-10-150-1-55.us-west-2.compute.internal30"[0] Transition from WaitActive to WaitQuorum,"",pid""v:"4759:msg"}0
,,":{"time""namemsg""::""crucible"":"[2] Transition from New to WaitActive,"","levelv""::300[1] Transition from LiveRepairReady to LiveRepair"2023-09-22T23:14:58.696566659Z",,"","v"hostname",name"":":time""ip-10-150-1-55.us-west-2.compute.internal",":crucible"":,2023-09-22T23:14:58.696590366Z""0,level"",hostname:"30:""pid":name"4759:ip-10-150-1-55.us-west-2.compute.internal"},""pid,"":time4759":}

"{crucible"{",msg":"""2023-09-22T23:14:58.696615146Zmsg"":,""hostname"level"::"[2] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) Active Active New ds_transition to WaitActive"30,"ip-10-150-1-55.us-west-2.compute.internal[0] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) WaitQuorum New New ds_transition to Active"",",v"pid""v:"4759::}0
,"0,"name{":""name":"msgcrucible"":,""crucible"level":,",30level":[2] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) Active Active WaitActive ds_transition to WaitQuorum30""time",:""v":0,2023-09-22T23:14:58.696647482Z",,,""time"hostname":"":nametime""::""""crucible2023-09-22T23:14:58.696669662Z"","ip-10-150-1-55.us-west-2.compute.internal"level,"":,302023-09-22T23:14:58.696671851Z"hostname"",pid":"hostname":":,"4759ip-10-150-1-55.us-west-2.compute.internal""ip-10-150-1-55.us-west-2.compute.internaltime"",:""pid"}:2023-09-22T23:14:58.696697924Z,""4759,"}pid"hostname
":
{:4759""}
ip-10-150-1-55.us-west-2.compute.internalmsg"":{",""pid"[0] Transition from WaitQuorum to Active:"4759msg"},
":"v{":0[2] Transition from New to WaitActive"",msg""name:"":","vcrucible"[2] Transition from WaitActive to WaitQuorum,"",":0level""v:"30:,0",name":""crucible"name",":level":,""crucibletime"",":level"":30302023-09-22T23:14:58.696754892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"}time"
,":"time{":"2023-09-22T23:14:58.696768481Z",""2023-09-22T23:14:58.696768949Zmsg"",:""hostname":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) Active Active WaitQuorum ds_transition to Active","v":0,"name"::""crucible","level":30[2] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.696848906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"time":"}
2023-09-22T23:14:58.696857251Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal",""pid":msg"4759:}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.696892553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":""[2] Transition from WaitQuorum to Active","v":0,"name":"crucible"[2] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) Active Active WaitQuorum ds_transition to Active,""level":,30"v":0,"name":"crucible","level":30[1] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) Active New New ds_transition to WaitActive,""time":","2023-09-22T23:14:58.696918446Zv"":,0","hostname"name:"":"crucible","ip-10-150-1-55.us-west-2.compute.internal"level",:"30,"pid"time:"4759:"}
2023-09-22T23:14:58.696926301Z","hostname":"{,"time":""ip-10-150-1-55.us-west-2.compute.internal"msg2023-09-22T23:14:58.696940506Z"":",,""pid":hostname"test live_repair::repair_test::test_live_repair_enqueue_close ... 82297280-9384-4cb6-a5ef-2a47addce4c7 is now active with session: b2023c24-e7e0-43e8-ac1f-929ca1abdea3:""4759,ip-10-150-1-55.us-west-2.compute.internal"",v""pid:"0:,4759}ok
}"
name":"{crucible{"
",msg""level:"":"30[1] Transition from New to WaitActive",msg"":"v":0,"[2] Transition from WaitQuorum to Active"name,"":,""time"crucible":,""v":level02023-09-22T23:14:58.696996698Z",":"30name":",crucible"","level":hostname"30:","ip-10-150-1-55.us-west-2.compute.internaltime"":,""pid":47592023-09-22T23:14:58.697013145Z"}
,","time":{hostname""":"msg":"2023-09-22T23:14:58.697019081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internal","pid[1] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) Active Active Active ds_transition to Faulted"":,4759"pid"}v
"::0{,4759""name":msg""}crucible:"
,""level":30{"msg":"[1] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab is now active with session: 10aa236c-40fd-46b5-be15-bb668c9fdf95"crucible",,"",level"":time30"v":":0,"2023-09-22T23:14:58.697055949Z"name",":"hostname":"crucible",,"ip-10-150-1-55.us-west-2.compute.internal"level":"30,time"":pid"":4759}2023-09-22T23:14:58.697069019Z
","hostname"{:""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] Transition from Active to Faulted"},
"v":0,{,"""name"time:msg"":crucible""":",[1] Transition from WaitActive to WaitQuorum""level,""2023-09-22T23:14:58.697082294Z:v30":"0,","name"hostname":":"crucible","ip-10-150-1-55.us-west-2.compute.internallevel",:"30"time,":""pid":47592023-09-22T23:14:58.697106177Z","}
hostname,"":"time":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.697115777Z"",",{"pid":hostname"4759:"}
ip-10-150-1-55.us-west-2.compute.internal"",{"msg":""pid"msg:":4759"}
[1] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) Active Active Active ds_transition to Faulted"{,"[1] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) Active Faulted Active ds_transition to LiveRepairReady""msg",:""v"v"::00,","name[1] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) Active WaitQuorum New ds_transition to Activename":""":,""cruciblev"",:"0level,"":crucible"name30":,""level":crucible30,"time":"2023-09-22T23:14:58.697163236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
",","time":"level{":302023-09-22T23:14:58.697167606Z"",msg":""hostname":"[1] Transition from Faulted to LiveRepairReady"ip-10-150-1-55.us-west-2.compute.internal",","v"pid":,0",:time""name:"":"4759crucible2023-09-22T23:14:58.697193362Z"","}level,"":
hostname30":"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759}"
,"msg":{time":"""msg"2023-09-22T23:14:58.697219028Z:"","[1] Transition from Active to Faulted"hostname"[1] Transition from WaitQuorum to Active,"":",v"":vip-10-150-1-55.us-west-2.compute.internal"":,0",0pid","name:"4759:"}
crucible"",{"name":level""msg:"30:""crucible","level":30[1] 82297280-9384-4cb6-a5ef-2a47addce4c7 (b2023c24-e7e0-43e8-ac1f-929ca1abdea3) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,,""timename""::""crucible","2023-09-22T23:14:58.697255148Z"level":,30"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time"{,":""time"msg:"":"2023-09-22T23:14:58.697259369Z"2023-09-22T23:14:58.697268363Z",","hostname":hostname"[2] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) Active Active New ds_transition to WaitActive"":","v":ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal0",,""namepid""::"4759,crucible}"
,""level{"pid"":msg30"::"4759}
[1] Transition from LiveRepairReady to LiveRepair","v":,0","time"name:"":{"2023-09-22T23:14:58.697302642Z"crucible",","hostnamelevel""::30""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) Active Faulted Active ds_transition to LiveRepairReady",{","timemsg":"":""v":2023-09-22T23:14:58.697318501Z[2] Transition from New to WaitActive"",0,"",vhostname""::0",""name"nameip-10-150-1-55.us-west-2.compute.internal:"",":"crucible""pid,"":crucible"4759level"}:
30,"level{":30"msg":"No repair needed for extent 1","v":0,"name":",crucible""time,"":"level":302023-09-22T23:14:58.69735316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,","{time"time"":msg"":":2023-09-22T23:14:58.697364546Z"","hostname"2023-09-22T23:14:58.6973564Z":","[2] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) Active Active WaitActive ds_transition to WaitQuorumhostname""ip-10-150-1-55.us-west-2.compute.internal",,""v"pid:"0:,4759":",name"":"":"crucible"downstairs,""}level
":ip-10-150-1-55.us-west-2.compute.internal"30,"pid":4759}
,"time":"{2023-09-22T23:14:58.697399747Z",""hostnamemsg":"":"[1] Transition from Faulted to LiveRepairReady"ip-10-150-1-55.us-west-2.compute.internal",,""v":pid"0:,4759"name}
":"crucible","{level":30"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30{","msgtime":"":"2023-09-22T23:14:58.697433873Z",",hostname"Crucible stats registered with UUID: 0ee60585-2ce5-46cc-99f9-95940127d893"":","time":"v"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.697442867Z":"0,,"",hostname":pid"":"4759name"ip-10-150-1-55.us-west-2.compute.internal"},:
""pid"crucible":{,4759"level"}:"
msg"30:"{"msg":"[1] c5b1fe31-ed78-40c7-a076-a3fdfc0cb3ab (10aa236c-40fd-46b5-be15-bb668c9fdf95) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"[2] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) Active Active WaitQuorum ds_transition to Active"crucible",",v":"0level",":name":"30crucible","level":30,"time":"2023-09-22T23:14:58.697496383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"time}"
:,""time":"2023-09-22T23:14:58.697514676Z"{,2023-09-22T23:14:58.697517133Z""hostname":",""msg"ip-10-150-1-55.us-west-2.compute.internal"hostname:":"","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid"}Crucible 0ee60585-2ce5-46cc-99f9-95940127d893 has session id: 3bbe3d15-6556-4aef-8fef-434df4baecd5
:"4759,"{v":}0
,""name"msg":{:""crucible"","msg[1] Transition from LiveRepairReady to LiveRepair"":level,"""v"[2] Transition from WaitQuorum to Active"::,"0v,30"test live_repair::repair_test::test_live_repair_enqueue_repair_noop ... ":ok0,
"name":"name":crucible"","crucible"level":,30",level"":time":"302023-09-22T23:14:58.697589777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,","time":time"":"{2023-09-22T23:14:58.697608189Z"2023-09-22T23:14:58.697613189Z","hostname":",""msghostname":"ip-10-150-1-55.us-west-2.compute.internal"":"ip-10-150-1-55.us-west-2.compute.internal",,""pid":pid"[0] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) New New New ds_transition to WaitActive"4759:,"}4759
v"}:
{0,"name":"crucible"",msg":""level":300df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 is now active with session: 443b4f09-6d76-4302-8b47-0237fa76d564","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.69767551Z","hostname":","ip-10-150-1-55.us-west-2.compute.internaltime"",":"pid":47592023-09-22T23:14:58.697684279Z","}hostname":"
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""msg":[0] Transition from New to WaitActive"","v":0,"name":"crucible","level":[1] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) Active Active Active ds_transition to Faulted"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.697735938Z",","time":hostname":""2023-09-22T23:14:58.697741246Z"ip-10-150-1-55.us-west-2.compute.internal,""hostname":","pid":4759ip-10-150-1-55.us-west-2.compute.internal","}pid"
:4759}
{{"msg":""msg":"[1] Transition from Active to Faulted","v":[0] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) WaitActive New New ds_transition to WaitQuorum"0,,""vname""::"0crucible",","namelevel":"30:"crucible","level":30,"time":"2023-09-22T23:14:58.697801879Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid":2023-09-22T23:14:58.697805276Z"4759,"}
hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":"4759msg":"}
[1] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) Active Faulted Active ds_transition to LiveRepairReady","v":0,{"name":"crucible",""levelmsg":"":30[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.697853007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":""msg":"2023-09-22T23:14:58.697860652Z","[1] Transition from Faulted to LiveRepairReady"hostname",:test live_repair::repair_test::test_live_repair_enqueue_reopen ... ""okv"
ip-10-150-1-55.us-west-2.compute.internal":,"0pid",":name":"4759crucible"},"
level":30{"msg":"[0] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.697913502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.697927982Z"{,"hostname":""msg":ip-10-150-1-55.us-west-2.compute.internal"","pid":4759}
[1] 0df2f9b9-49ac-4d4a-a922-07b39a2dd1e6 (443b4f09-6d76-4302-8b47-0237fa76d564) Active LiveRepairReady Active ds_transition to LiveRepair",{"{v":0,"name":"""msg"crucible"msg:"",:"[0] Transition from WaitQuorum to Active"level"",:"30v":0Crucible stats registered with UUID: 33e28e69-0d9e-4c9f-b7a6-74c7045e411b","name":",crucible""v",":level":030,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.697976126Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"time":}"
2023-09-22T23:14:58.697984533Z",",hostname""{:"time":""msg"2023-09-22T23:14:58.69798927Z":"ip-10-150-1-55.us-west-2.compute.internal",","[1] Transition from LiveRepairReady to LiveRepair"pid",":hostname"4759v":"}:
ip-10-150-1-55.us-west-2.compute.internal"0,,""namepid"{:":4759""crucible"},msg":"
"level":[1] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) Active New New ds_transition to WaitActive30","v"{:0,"name":"crucible"","msglevel"":":30,"time":"Crucible 33e28e69-0d9e-4c9f-b7a6-74c7045e411b has session id: 17410f8a-bf36-4608-9f9d-0991c783a7cc"2023-09-22T23:14:58.698053348Z,""v":,0","hostname":name"":"crucible","ip-10-150-1-55.us-west-2.compute.internallevel"":,"30pid,":"4759time":}"
2023-09-22T23:14:58.698064813Z","hostname":{"ip-10-150-1-55.us-west-2.compute.internal"","msg"pid",":"time":":Repair for extent 1 s:0 d:[ClientId(2)]"47592023-09-22T23:14:58.698085472Z","},v"":
0{,""name":"msg"crucible":","level":30[1] Transition from New to WaitActive","v":0,"hostnamename""::""crucible","level":ip-10-150-1-55.us-west-2.compute.internal"30,"pid":4759}
,"time":"2023-09-22T23:14:58.698136582Z","{hostname":"ip-10-150-1-55.us-west-2.compute.internal",""pidmsg","":time"":"2023-09-22T23:14:58.698147423Z","hostname":"[0] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) New New New ds_transition to WaitActive"ip-10-150-1-55.us-west-2.compute.internal",,""v"pid"::04759,"name":"}
crucible","level":30:{4759,""":msg"":"downstairs"}
[1] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) Active WaitActive New ds_transition to WaitQuorum","v":0,","timename":"":"crucible"2023-09-22T23:14:58.698191068Z","level",":hostname":"30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive",,""v"time":":0,"2023-09-22T23:14:58.698217701Z"name":","crucible"hostname":","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":"[1] Transition from WaitActive to WaitQuorum",2023-09-22T23:14:58.698246038Z""v",:"0hostname,"":"name":"crucible","ip-10-150-1-55.us-west-2.compute.internal"level,""{pid"msg":"":4759Crucible stats registered with UUID: 6b3331ad-9fa4-46f8-ac51-e5e343fd372f"}
,"v":0,"name":"{crucible","level":30"msg":"[0] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.698297864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"time":}"
{"msg":"2023-09-22T23:14:58.698310132Z"Crucible 6b3331ad-9fa4-46f8-ac51-e5e343fd372f has session id: 6f5b1cf0-5769-42d2-849f-2f27f3b8dfa7","v",::0"30,hostname"":name"":"crucible","ip-10-150-1-55.us-west-2.compute.internal"level":,"30pid":4759}
,{"time":""msg,"":time"":"[0] Transition from WaitActive to WaitQuorum"2023-09-22T23:14:58.698357384Z",,""v":hostname":0","name":"ip-10-150-1-55.us-west-2.compute.internalcrucible"",","pid":level"4759:30}
{"msg":"[0] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) New New New ds_transition to WaitActive","v":,"0,time"":"name":"crucible"2023-09-22T23:14:58.698386293Z",",level":2023-09-22T23:14:58.698353512Z30","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.698410863Z"{,"hostname":"""ip-10-150-1-55.us-west-2.compute.internal"hostname":"test live_repair::repair_test::test_live_repair_enqueue_repair_repair ... ,ok"
pid":4759msg}
{":""msg":"ip-10-150-1-55.us-west-2.compute.internal[0] Transition from New to WaitActive"",[1] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) Active WaitQuorum New ds_transition to Active"","v":0,"name"v":":crucible"0,","level"name,"":pid"":crucible"4759,"level":}30
{:"30msg":","time":"[0] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) WaitQuorum New New ds_transition to Active"2023-09-22T23:14:58.698486335Z",",v"":hostname"0:","name":"crucible"ip-10-150-1-55.us-west-2.compute.internal",","level"pid:":304759}
{"msg":,""time":"2023-09-22T23:14:58.698496651Z",,"[0] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) WaitActive New New ds_transition to WaitQuorumtime":""2023-09-22T23:14:58.69850831Z,""v",:"0,hostname"":"name":"crucible","ip-10-150-1-55.us-west-2.compute.internal"level,"":pid30":4759}
"{hostname"":"msg":,""time":"ip-10-150-1-55.us-west-2.compute.internal"[0] Transition from WaitQuorum to Active"2023-09-22T23:14:58.698533909Z,"",,""pid"hostname"::"4759ip-10-150-1-55.us-west-2.compute.internal}"
,"pid":4759}
v":0,"{name":"crucible""msg,"":"level":30[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","{level":30"msg":"[1] Transition from WaitQuorum to Active","time,",:""2023-09-22T23:14:58.69857772Ztime":"","2023-09-22T23:14:58.698583845Z"hostname",:""hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid""":,"v"pid4759":}:
04759,"}name"
{:"crucible","level":30,"time":"2023-09-22T23:14:58.698636666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"":4759msg":"}
[0] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) WaitQuorum New New ds_transition to Active","v":{0,"{name""msg"::""msg":"crucible","level":30[1] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) Active New New ds_transition to WaitActive","v":0,""name":"crucible","level":30[2] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.69867768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal,"",time"":pid"":47592023-09-22T23:14:58.698686528Z"}
{,"time":"2023-09-22T23:14:58.698691629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"{msg":""[0] Transition from WaitQuorum to Activemsg":"","v":0[2] Transition from New to WaitActive,""name","v":0,"name:":""crucible"crucible,"","level":30level":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,"4759time":"}
2023-09-22T23:14:58.698758286Z,""time":","hostname2023-09-22T23:14:58.698760116Z"{",":""msgip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30":","time":"[1] Transition from New to WaitActive",2023-09-22T23:14:58.698804772Z""v",:"0hostname,"":"name":"crucible",ip-10-150-1-55.us-west-2.compute.internal"",level"":pid":304759}
{"msg":"hostname":[1] Transition from New to WaitActive","time":"2023-09-22T23:14:58.698824862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible",,""v"level:":030,"name":"crucible","level":30,"time,"":"time":"{2023-09-22T23:14:58.698864576Z"2023-09-22T23:14:58.698867733Z",","hostname"hostname:"":""msg"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""ip-10-150-1-55.us-west-2.compute.internalpid""":pid"4759:4759},
}"
{{"msg":"[1] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) Active WaitActive New ds_transition to WaitQuorum","v"pid:"0":msg,"":name4759"}
:"{crucible","level":"30"msg":"[1] Transition from WaitActive to WaitQuorum","[2] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) Active Active WaitActive ds_transition to WaitQuorum"v",:"0v":,0,,"name""time":":"2023-09-22T23:14:58.698980848Z"crucible",","hostname"level"::"30:"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}Crucible stats registered with UUID: 8022d443-8555-401b-9593-47c602f81c95"
,"{time":""2023-09-22T23:14:58.699004568Z"msg,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":""crucible","level":name"30:"crucible","level":30,"time":"2023-09-22T23:14:58.69904335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
":"{[1] Transition from WaitActive to WaitQuorum","","msg":"vtime""::"0,"name"2023-09-22T23:14:58.699047438Z:""crucible,"","hostname"level"::","ip-10-150-1-55.us-west-2.compute.internalv"30:0,"name":"crucible","[2] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) Active Active WaitQuorum ds_transition to Activelevel"":30,"v":0,",name":""crucible","level":30time":"2023-09-22T23:14:58.699075967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":"4759,}"
{,"msg":"[1] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) Active WaitQuorum New ds_transition to Active","v":"0time",:"",name2023-09-22T23:14:58.699085396Z"""time"::""2023-09-22T23:14:58.699091652Z"pid,"":hostname":"4759}ip-10-150-1-55.us-west-2.compute.internal
","pid":4759}
{{"msg":""msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30[1] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":crucible"30,"level":30,,""time":"hostname":"2023-09-22T23:14:58.699201665Z","ip-10-150-1-55.us-west-2.compute.internal"hostname":","pid":4759ip-10-150-1-55.us-west-2.compute.internal","}pid
":4759}
{{""msg"msg:"":"0ee60585-2ce5-46cc-99f9-95940127d893 is now active with session: 9ee1ca43-1cdd-48fb-af14-ac82b4ae106dCrucible 8022d443-8555-401b-9593-47c602f81c95 has session id: bc94380a-dffd-40a5-bb00-352c87a8d400"",","v"v:":00,,"",name"name"":":"crucible"crucible,"",level"":level30":time30":"2023-09-22T23:14:58.69920968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg",,""time"time:"":":"2023-09-22T23:14:58.699251626Z"2023-09-22T23:14:58.699253172Z",,""[1] Transition from WaitQuorum to Active"hostnamehostname""::"","v":0,"name":"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:"4759:crucible"4759,}
,"{time":""msg":2023-09-22T23:14:58.699208443Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal"[1] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) Active Active Active ds_transition to Faulted",","pidv"":"0level":30,"name":"crucible","level":30,"time":":2023-09-22T23:14:58.699307386Z","4759hostname},"
time}""{:":"2023-09-22T23:14:58.699312888Z""msg":,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[0] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) New New New ds_transition to WaitActive"}
,"v":0{,"name":""msgcrucible"":","level":30[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.699351779Z","hostname":"
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"{msg":"""msg"msg"::""[1] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) Active Faulted Active ds_transition to LiveRepairReady","v":[1] Transition from WaitQuorum to Active"0[2] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) Active Active New ds_transition to WaitActive,",,""vname":"crucible","level":30,"time":"2023-09-22T23:14:58.699399162Z","hostname":""ip-10-150-1-55.us-west-2.compute.internal:"0,",pid"":name4759":"}
"crucible{v":"0msg":","name":"[1] Transition from Faulted to LiveRepairReadycrucible"",","levelv":"0:,"30name":"crucible",","level":30"level":30,"time":",2023-09-22T23:14:58.699447664Z"",time","time":"2023-09-22T23:14:58.699440152Z","hostname":","time":ip-10-150-1-55.us-west-2.compute.internal"","pid":2023-09-22T23:14:58.699456691Z"4759,"hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":hostname4759":"}
{ip-10-150-1-55.us-west-2.compute.internal","pid"{:4759"}
":"2023-09-22T23:14:58.699347119Z","hostname":""msg"ip-10-150-1-55.us-west-2.compute.internal:"","pid":4759}
[1] 0ee60585-2ce5-46cc-99f9-95940127d893 (9ee1ca43-1cdd-48fb-af14-ac82b4ae106d) Active LiveRepairReady Active ds_transition to LiveRepair","v":{0,"name":"crucible""msg",:""{level":30[0] Transition from New to WaitActive",""msgv":","time":"2023-09-22T23:14:58.699519214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
:"{[2] Transition from New to WaitActive",""msg"v:"":0,"name":"crucible"[1] Transition from LiveRepairReady to LiveRepair",",level""v":00,"name":","crucible"name",:""level"crucible:"30,"level":30,"time":":30msg":"2023-09-22T23:14:58.699560692Z,"time":"2023-09-22T23:14:58.69957927Z",[2] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) Active Active New ds_transition to WaitActive""hostname":"ip-10-150-1-55.us-west-2.compute.internal,"","v"pid"::47590}
{"msg":"[2] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible",,""levelname":"crucible","level":30,"time":""2023-09-22T23:14:58.699562266Z":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,,""time":"{2023-09-22T23:14:58.699636033Z"",msg"":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","[0] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) WaitActive New New ds_transition to WaitQuorum"pid",":v":47590,"name":"crucible"},"
level"time"::"302023-09-22T23:14:58.699643188Z","{hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"":"4759msg,"}
,"time":"2023-09-22T23:14:58.699688461Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759"}
:"{{"[2] Transition from New to WaitActivemsg""msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30:,""time":"2023-09-22T23:14:58.699732552Z"[2] Transition from WaitActive to WaitQuorum",",hostname""v":":0,"name":"crucible","ip-10-150-1-55.us-west-2.compute.internallevel"":,"30hostnamepid":"4759:"}
ip-10-150-1-55.us-west-2.compute.internal{"msg":","time":"2023-09-22T23:14:58.699759474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","[0] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) WaitQuorum New New ds_transition to Active"pid":,4759"v"}:
{"msg":"[2] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) Active Active WaitQuorum ds_transition to Active","v":0,"name":"0crucible",","namepid""::""crucible4759",",}"
level":level30":30",","time"v":":2023-09-22T23:14:58.699807051Z",0,",hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.699834958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"6b3331ad-9fa4-46f8-ac51-e5e343fd372f is now active with session: b654e397-ac15-4a47-a3bf-f5ba3d4d7f25","v":0,"name":""crucible","timelevel":30"name":"crucible",""level"::"30,"2023-09-22T23:14:58.699804837Ztime"":","2023-09-22T23:14:58.699868741Z"hostname",:""hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internal"pid",:"pid4759":4759}}
,"time":"2023-09-22T23:14:58.699879825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{""msgmsg":"[2] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30":","time":"[1] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) Active Active Active ds_transition to Faulted"2023-09-22T23:14:58.699936874Z",","v":hostname":"0,"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible,"","pidlevel":"4759:30}
{
,""msg":time"{:"msg":"[0] Transition from WaitQuorum to Active","v":0","name":"2023-09-22T23:14:58.699969424Z"crucible",","hostname"level:""ip-10-150-1-55.us-west-2.compute.internal","pid:":304759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":","time":"test live_repair::repair_test::test_live_repair_flush_is_flush ... 2023-09-22T23:14:58.700003736Zok
","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"crucible","level":30[1] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30","time":"[2] Transition from WaitActive to WaitQuorum"2023-09-22T23:14:58.700044982Z",",hostname,"":v":0,"name":"crucible","level":30","time":"ip-10-150-1-55.us-west-2.compute.internal","pid"2023-09-22T23:14:58.700070545Z":4759,"}hostname
{""time"msg"::""2023-09-22T23:14:58.700051897Z","hostname":"[1] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) Active Faulted Active ds_transition to LiveRepairReady"ip-10-150-1-55.us-west-2.compute.internal",,""vpid":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.700140452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
":4759}
{{""msgmsg":"":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30"msg":"[2] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) Active Active WaitQuorum ds_transition to Active"[1] Transition from Faulted to LiveRepairReady,""v":0,,",v""name"time""::"":crucible"02023-09-22T23:14:58.700298282Z,"",",namelevel"":"":crucible"hostname,30""level"::30"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":"2023-09-22T23:14:58.700340785Z","hostname"":msg"",":ip-10-150-1-55.us-west-2.compute.internaltime""",:""pid":47592023-09-22T23:14:58.700338285Z}"[1] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) Active WaitActive New ds_transition to WaitQuorum"
,,""v":{0hostname,"""msg":":name":""crucible","level":ip-10-150-1-55.us-west-2.compute.internal30",[1] 6b3331ad-9fa4-46f8-ac51-e5e343fd372f (b654e397-ac15-4a47-a3bf-f5ba3d4d7f25) Active LiveRepairReady Active ds_transition to LiveRepair"",pid"":v"4759:0,"}name
":"crucible","level{":30,"time{":""msg":""2023-09-22T23:14:58.700389439Z"msg":","Crucible stats registered with UUID: 7fad8992-b20b-4cb6-990d-67be4c403b6f"hostname"[2] Transition from WaitQuorum to Active,":,v""time:":0"",","2023-09-22T23:14:58.700405911Z""name,"":"hostname"crucible":,""level":ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal30"",v",""pid"pid"::4759:4759}0
,",time"}"{:"name""2023-09-22T23:14:58.700432715Zmsg"":,
""hostname":":[1] Transition from LiveRepairReady to LiveRepaircrucible""{,"ip-10-150-1-55.us-west-2.compute.internal""v,","msglevel"""":pid0",:":name4759":}"
crucible""{,":level[1] Transition from WaitActive to WaitQuorum30"":msg30":"","v":0,"name":"Crucible 7fad8992-b20b-4cb6-990d-67be4c403b6f has session id: c32553ed-11d5-4914-9299-facbcae54f86crucible"",","levelv":":0,,""time"name:"":30"2023-09-22T23:14:58.700489131Z"crucible",,""levelhostname""::"30,"timeip-10-150-1-55.us-west-2.compute.internal":"","pid":2023-09-22T23:14:58.700488217Z4759"}
,,"{"time""msg:"":,"2023-09-22T23:14:58.700507367Z"",Create read repair deps for extent 1time"hostname"::""""hostname,"":v""ip-10-150-1-55.us-west-2.compute.internal":ip-10-150-1-55.us-west-2.compute.internal0",,""2023-09-22T23:14:58.700504721Z","pidname"":,4759:"}"
crucible"hostname"pid":":{,4759""levelmsg""::40"ip-10-150-1-55.us-west-2.compute.internal"},"
pid"[0] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) New New New ds_transition to WaitActive,"",time":"v{4759"::"0},2023-09-22T23:14:58.700560084Z""name"",:"
msg"hostname"":crucible"{:""ip-10-150-1-55.us-west-2.compute.internal",","level""pid:"msg":30475933e28e69-0d9e-4c9f-b7a6-74c7045e411b is now active with session: 1c5e44dd-707c-49ea-88a0-d303601a440f":"},"
v"[1] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) Active WaitQuorum New ds_transition to Active,""{time":",msg"":"2023-09-22T23:14:58.700593882Z:""v":0IO Read  1004 extent 0 added deps 1,"",,0"hostnamev"":,""":name"ip-10-150-1-55.us-west-2.compute.internal0",,"":name":""crucible"pidname"":crucible",,:4759""}crucible
"level,----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:1  downstairs:1
"levelGW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
{    1 AckReady  1000   Write       1"   Newmsg"   New":40   New  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           1   "   1   ",level   1   ::   3
    Sent       "   0   "   0   time"   0   :   0
    Done          0      0      0   "   0
    Skipped    :2023-09-22T23:14:58.70071889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
   0   30   0      0      0
    Error         0      0      0      0
Last Flush: 0 0 0 
"30[0] Transition from New to WaitActive"Downstairs last five completed:,",
","time":"Upstairs last five completed:  2023-09-22T23:14:58.700817791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"
time":"2023-09-22T23:14:58.700801546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"pid":4759}
{""msg":"msg":"[1] Transition from WaitQuorum to Active","v":v0",":name"0:","crucible"name",:""level"crucible":,"30[1] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) Active Active Active ds_transition to Faulted"level":30,"v":0,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:14:58.700942619Z2023-09-22T23:14:58.700944516Z"",,""hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid:",":47594759time"}}

{test live_repair::repair_test::test_live_repair_repair_read_push ... "{msg"::""msg":""ok2023-09-22T23:14:58.700951146Z"
[2] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) Active Active New ds_transition to WaitActive[0] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) WaitActive New New ds_transition to WaitQuorum"",,""vv""::00,,,""name"name:"":""cruciblecrucible""hostname,,""level"level:"30:"30:"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,,""timetime""::""2023-09-22T23:14:58.701013888Z2023-09-22T23:14:58.701012536Z"",{,""hostnamehostname""::"""msg":ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:"4759:"4759}
}[1] Transition from Active to Faulted"
{,"v":"{0msg""msg:"":","name":"[0] Transition from WaitActive to WaitQuorum[2] Transition from New to WaitActive"",,""vv""::00,,""name"name:"":crucible""crucible",crucible""level":,30",level""level"::3030,"time":"2023-09-22T23:14:58.70106546Z","hostname",:""time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.701070317Z,"",pid"":4759hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
",msg{""":"msgtime":":""2023-09-22T23:14:58.701071377Z",[2] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) Active Active WaitActive ds_transition to WaitQuorum"",[0] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) WaitQuorum New New ds_transition to Active""v",:"0v,"":hostname":name0",:""name"crucible":,"""levelcrucible"":,30"levelip-10-150-1-55.us-west-2.compute.internal"":,30"pid":4759}
,,""timetime""::""2023-09-22T23:14:58.701113248Z2023-09-22T23:14:58.701116874Z"",","hostname":hostname""{:"ip-10-150-1-55.us-west-2.compute.internal","pid":4759ip-10-150-1-55.us-west-2.compute.internal","}pid
":"4759{msg"}"
msg"::{"""msg":"[2] Transition from WaitActive to WaitQuorum","v":[0] Transition from WaitQuorum to Active0",","namev""::"0[1] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) Active Faulted Active ds_transition to LiveRepairReady"crucible,"",name""level:"":,"crucible30"v,""level"::300,"name":"crucible","level",":time"30:"2023-09-22T23:14:58.701160068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible",",level"":time"30:"2023-09-22T23:14:58.701164997Z",","time"hostname":":"2023-09-22T23:14:58.701172752Z"ip-10-150-1-55.us-west-2.compute.internal",,""timepid""::"4759,"}2023-09-22T23:14:58.701197858Z"
,hostname""{:"hostname"":msg"":"ip-10-150-1-55.us-west-2.compute.internal","pid"ip-10-150-1-55.us-west-2.compute.internal":,4759"[1] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) Active New New ds_transition to WaitActive"pid":,"}v4759":}0
,"
name{":""crucible"msg":,""level":30{[2] Transition from WaitQuorum to Active","v":0,"name"":"msg":"crucible","level":[1] Transition from Faulted to LiveRepairReady30,""time",":"v":02023-09-22T23:14:58.701238324Z,""name":",,""hostname"time:"":"crucible","ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.701247664Z"",,""pid"hostname:"4759:level""}:
ip-10-150-1-55.us-west-2.compute.internal"30,{"pid":4759"msg"}:
"{[1] Transition from New to WaitActive",""msgv""::"0,"name":"crucible","level":308022d443-8555-401b-9593-47c602f81c95 is now active with session: 9a974847-a7c7-4136-88bb-99c862b855df","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.701273742Z",",time":""hostname":",2023-09-22T23:14:58.701284913Z""time",:"ip-10-150-1-55.us-west-2.compute.internal"hostname"","2023-09-22T23:14:58.701290613Z:"",pid"":ip-10-150-1-55.us-west-2.compute.internalhostname4759"":","pid"}ip-10-150-1-55.us-west-2.compute.internal:"4759,"
}{"msg":"[1] 33e28e69-0d9e-4c9f-b7a6-74c7045e411b (1c5e44dd-707c-49ea-88a0-d303601a440f) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30
{"msg":"[1] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) Active WaitActive New ds_transition to WaitQuorum","v":0,,""nametime":":""crucible2023-09-22T23:14:58.7013447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.701395395Z",{"hostname":""ip-10-150-1-55.us-west-2.compute.internal"msg",:""pid":4759}
Crucible stats registered with UUID: f40c9366-ba51-49e8-a7ac-2e8d70c8171b","v":0,"name":"crucible","level":30","level":30,","time"time:"":"pid2023-09-22T23:14:58.701438519Z2023-09-22T23:14:58.701434469Z""",,"":hostnamehostname"":4759:""}ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"",
"pid"pid:":47594759}}

{{"{msg":""msg":""[1] Transition from WaitActive to WaitQuorum",msg"":vCrucible f40c9366-ba51-49e8-a7ac-2e8d70c8171b has session id: 6ab93a12-24ca-44b2-9736-92c62af6e777"":,"0v,""name:"0:,"""namecrucible"":","crucible"level",:"30level":30[1] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:14:58.701493358Z2023-09-22T23:14:58.701493996Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{{"msg":""msg":","time":"[1] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) Active WaitQuorum New ds_transition to Active"[0] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) New New New ds_transition to WaitActive",",v""v:"0:,0",name""2023-09-22T23:14:58.70150495Zname:"":"crucible"crucible",,""level":level30":"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,,""timetime""::""{2023-09-22T23:14:58.701541222Z2023-09-22T23:14:58.701539743Z"",,""hostname"hostnamemsg"""::"":"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759[1] Transition from Active to Faulted"}}

{,"v"{":msg"0:""msg":","[1] Transition from WaitQuorum to Activename"[0] Transition from New to WaitActive"",",v""v:"0:,0",name""::""namecrucible""crucible:"",,crucible""level,"""level:"30:level"30:30,",time"":time"":"2023-09-22T23:14:58.701598034Z"2023-09-22T23:14:58.701599736Z",","hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:"4759:,}4759
}"
{time":"{"msg":""msgtest live_repair::repair_test::test_live_repair_no_repair_yet ... 2023-09-22T23:14:58.701601002Z"ok,
"[2] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) Active Active New ds_transition to WaitActive:""","hostname"v":":[0] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) WaitActive New New ds_transition to WaitQuorum0",","name"v:":"0ip-10-150-1-55.us-west-2.compute.internal",crucible""name":,"",levelcrucible"":,30""level"pid"::304759}
,"time":","time":"2023-09-22T23:14:58.701655828Z"{,2023-09-22T23:14:58.701660036Z"",hostname"":"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{[1] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) Active Faulted Active ds_transition to LiveRepairReady""{msg",":msg"":""[0] Transition from WaitActive to WaitQuorum"v",[2] Transition from New to WaitActive""v,""v:"0:,0",name""name:"":"crucible"crucible":,0",,level""level:"30":name30":"crucible","level":30,"time":","time":"2023-09-22T23:14:58.701704711Z","hostname2023-09-22T23:14:58.701706869Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"}pid
",:"{4759time":"}"
msg":"{2023-09-22T23:14:58.70171076Z"","msg":"hostname":"[2] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) Active Active WaitActive ds_transition to WaitQuorum",ip-10-150-1-55.us-west-2.compute.internal"","v[0] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) WaitQuorum New New ds_transition to Active"":,0",v"":name0",:""cruciblename"":,""pid"levelcrucible"":,30":level":475930}
,","time"time:""{:"2023-09-22T23:14:58.701759009Z","2023-09-22T23:14:58.701761588Z"hostname","":msg":"hostname"":"[1] Transition from Faulted to LiveRepairReady"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,"",pidpid""::47594759"}}

{v":{"0msg""msg:"":,""name"[2] Transition from WaitActive to WaitQuorum"[0] Transition from WaitQuorum to Active",:",""vv""::00,,""crucible"namename,"""::""level"cruciblecrucible"",,""level"level:"30::3030,,""timetime""::""2023-09-22T23:14:58.701813635Z2023-09-22T23:14:58.701814764Z"",,""hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid:"4759:,"}4759
}time":"{
"2023-09-22T23:14:58.701815695Z"msg{":,"""msg"hostname"::""ip-10-150-1-55.us-west-2.compute.internal","pid"[1] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active New New ds_transition to WaitActive":4759,"v"}[2] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) Active Active WaitQuorum ds_transition to Active:"0,,""v"name:"0:,""cruciblename"",:""level"crucible:"30,
"level":30{"msg":","time":"2023-09-22T23:14:58.701862406Z"[1] 8022d443-8555-401b-9593-47c602f81c95 (9a974847-a7c7-4136-88bb-99c862b855df) Active LiveRepairReady Active ds_transition to LiveRepair,"",hostname"":time",""v":ip-10-150-1-55.us-west-2.compute.internal0":,"","pid2023-09-22T23:14:58.701867762Z"":,name"4759":}hostname
"":{"crucible","level"":msgip-10-150-1-55.us-west-2.compute.internal"":,""30pid":[1] Transition from New to WaitActive4759",}"
v":0,"{name":""cruciblemsg"":,""level":30[2] Transition from WaitQuorum to Active","v":0,"name":"crucible",,""timelevel"":",:"30time":"2023-09-22T23:14:58.701901577Z","2023-09-22T23:14:58.701909561Z"hostname",:""hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internal",,""time":pid"":pid"47592023-09-22T23:14:58.701919327Z"}:,
"4759hostname{":"}
ip-10-150-1-55.us-west-2.compute.internal""msg,"":"pid":4759{}
"[1] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active WaitActive New ds_transition to WaitQuorum"{msg,"""vmsg""::0",:""name":"crucible","[1] Transition from LiveRepairReady to LiveRepair"level7fad8992-b20b-4cb6-990d-67be4c403b6f is now active with session: e14ba783-7271-41eb-9b62-8aeb270fd61b"":,30","v":v":00,,""namename":"":,""cruciblecrucible""time,"":level"",":level":302023-09-22T23:14:58.701963321Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{,"time"2023-09-22T23:14:58.701981956Z""msg:","":"2023-09-22T23:14:58.701983061Zhostname""[1] Transition from WaitActive to WaitQuorum:","",hostname":""ip-10-150-1-55.us-west-2.compute.internal"v",:"0pid,"":name"4759:"}ip-10-150-1-55.us-west-2.compute.internal"
crucible",",{"pid""level:"msg:"30:"4759}
{[1] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) Active Active Active ds_transition to Faulted",""v":,0",msg":"timename""::"""crucible"2023-09-22T23:14:58.702016694Z",,""level"hostname:"30:"Write to Extent 0:2:9 under repair","v":0,"nameip-10-150-1-55.us-west-2.compute.internal"":","crucible"pid,","":level4759time"}:
"":40{2023-09-22T23:14:58.702035899Z",""hostname"msg:"":"ip-10-150-1-55.us-west-2.compute.internal","pid":,"4759time":"}[1] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active WaitQuorum New ds_transition to Active
"2023-09-22T23:14:58.702050805Z",{","v""msg:"0:"hostname":"[1] Transition from Active to Faulted,"",name""v:"":0crucible,""ip-10-150-1-55.us-west-2.compute.internalname,"":level"":crucible30"",,""pid"level"::304759}
,"time":{{""2023-09-22T23:14:58.70208067Z,"",time"":"hostname"msg"2023-09-22T23:14:58.702086024Zmsg:"":"",":"hostname"Write 1:0:9 past extent under repair 0""ip-10-150-1-55.us-west-2.compute.internal:",",""Crucible stats registered with UUID: c8868e06-7e14-4a37-b718-8f66383d6b07v"":ip-10-150-1-55.us-west-2.compute.internalpid"",:"4759pid"}0
:,,""v"name:"0:,""{4759namecrucible"":,""crucible"level",:""}msg
"level40":{:"30"msg":[1] Transition from WaitQuorum to Active"","v":0,"name":"crucible","level":30[1] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) Active Faulted Active ds_transition to LiveRepairReady","v":0,",name"":"time":"crucible","2023-09-22T23:14:58.702133987Z"level,"":time30":,,""hostnametime"":":""ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:14:58.702142782Z2023-09-22T23:14:58.70213642Zpid"":,4759","hostname}"
:","hostname"{ip-10-150-1-55.us-west-2.compute.internal":",msg""pid:"":"4759Write 1:1:9 past extent under repair 0",}"
v":ip-10-150-1-55.us-west-2.compute.internal"0{,",name""msg""pidtime""::"4759"::""2023-09-22T23:14:58.702154321Z}"crucible
,"Crucible c8868e06-7e14-4a37-b718-8f66383d6b07 has session id: e17a1d32-1fd2-4f6a-9143-a7392fe78df3,""{level,""""v:"40:hostnamemsg""::""0,"name"ip-10-150-1-55.us-west-2.compute.internal":","crucible"pid":[2] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active Active New ds_transition to WaitActive4759",,""time"level:"":,}"
v"302023-09-22T23:14:58.702204826Z"{:,0",msg"""name:":""hostname"crucible":",[1] Transition from Faulted to LiveRepairReady""level",:"30v",ip-10-150-1-55.us-west-2.compute.internal""time,"":pid""::2023-09-22T23:14:58.702224011Z4759"0},
,",name"":{"time""hostname""msg:"":"crucible:"","ip-10-150-1-55.us-west-2.compute.internalIO Write 1004 on eur 0 Added deps 1"2023-09-22T23:14:58.702235715Zlevel"",:"30hostname","":pid",:"4759v":"0}
,",ip-10-150-1-55.us-west-2.compute.internal""{name":"",time"":pid"":cruciblemsg2023-09-22T23:14:58.702260212Z4759""":,"",}level"
"[0] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) New New New ds_transition to WaitActive:"40,{hostname""":msg""v":ip-10-150-1-55.us-west-2.compute.internal:"","0pid[2] Transition from New to WaitActive"":,4759",,""}v"
:timename"":0{,":crucible""",msg""name:"":"2023-09-22T23:14:58.70228638Z""crucible",,"level"hostname:"30:""[1] 7fad8992-b20b-4cb6-990d-67be4c403b6f (e14ba783-7271-41eb-9b62-8aeb270fd61b) Active LiveRepairReady Active ds_transition to LiveRepairlevel"":,30"ip-10-150-1-55.us-west-2.compute.internal"v",:"0pid",:,4759""}time
":,name""time:"""crucible:"","2023-09-22T23:14:58.70231671Z"2023-09-22T23:14:58.702319093Zlevel",","":hostname":"hostname30":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internal"pid":,4759"pid"}:,4759"
time}"
:{"{"2023-09-22T23:14:58.702340767Z"msg",:"""hostname"msg":":"[0] Transition from New to WaitActive",ip-10-150-1-55.us-west-2.compute.internal"[2] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active Active WaitActive ds_transition to WaitQuorum,"""pid,"":v4759":}0
,"v"name"{:":"crucible"0,msg"",level:"":"30name":"[1] Transition from LiveRepairReady to LiveRepair"crucible",","vlevel""::030,"name":","crucible"time",:""level":302023-09-22T23:14:58.70238446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time"{:",""2023-09-22T23:14:58.702396157Zmsg""time",:"":"hostname":"[2] Transition from WaitActive to WaitQuorum"2023-09-22T23:14:58.702392508Z",ip-10-150-1-55.us-west-2.compute.internal""v,"":,"0pid,hostname"""name:"4759:"}:"crucible
"ip-10-150-1-55.us-west-2.compute.internal",","level":pid30":4759}
{,"time":""msg":2023-09-22T23:14:58.702431939Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":[0] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) WaitActive New New ds_transition to WaitQuorum"4759,"}v":
0,"name":"crucible"{,"level":30"msg":"[2] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.702469682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.702482781Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":[0] Transition from WaitActive to WaitQuorum"4759,"v":}0
test live_repair::repair_test::test_live_repair_repair_write_push ... {,ok"
msg"":"name":"crucible"[2] Transition from WaitQuorum to Active",","level":v":300,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.702534142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{"2023-09-22T23:14:58.702538227Zmsg":"","hostname":"[0] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) WaitQuorum New New ds_transition to Active","vip-10-150-1-55.us-west-2.compute.internal"":,0","pidname":""crucible":,4759"level}":
30{"msg":"f40c9366-ba51-49e8-a7ac-2e8d70c8171b is now active with session: b7f69b5e-7fe1-4604-9c09-0d5b9217b341",,""timev":"0:","name":"2023-09-22T23:14:58.702575395Z"crucible",","hostname"level"::"30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.702597986Z","hostname":""ip-10-150-1-55.us-west-2.compute.internalmsg":"","pid":4759[0] Transition from WaitQuorum to Active"}
,"v":0,"{name":"crucible"",msg":""level":30[1] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.70262871Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:14:58.702637271Z"pid",":hostname":"4759ip-10-150-1-55.us-west-2.compute.internal"},"
pid":4759}
{{""msgmsg":"":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30[1] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.70267502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":""msg":"2023-09-22T23:14:58.702682796Z","hostname":"test live_repair::repair_test::test_live_repair_send_io_write_below ... [1] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active Faulted Active ds_transition to LiveRepairReadyok",
"ip-10-150-1-55.us-west-2.compute.internal"v",":pid"0:,"4759name":"crucible}"
,"level":30{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.702727196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.702740239Z",""hostname":"msg":"ip-10-150-1-55.us-west-2.compute.internal","[1] Transition from Faulted to LiveRepairReadypid"":,4759"v":}0
,"name":"crucible","{level":30"msg":"[1] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) Active WaitActive New ds_transition to WaitQuorum","v":0,",time":""name"2023-09-22T23:14:58.702776106Z":,""hostname":"crucible","ip-10-150-1-55.us-west-2.compute.internallevel"":,30"pid":4759}
{"msg":","time":"2023-09-22T23:14:58.702797661Z"[1] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active LiveRepairReady Active ds_transition to LiveRepair,"","hostname":v"":0,"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible",,""pid"level"::304759}
{"msg":"[1] Transition from WaitActive to WaitQuorum",",v"":time":0","name":"2023-09-22T23:14:58.702828119Z"crucible",,""hostname"level":":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time"":msg"":"2023-09-22T23:14:58.702854089Z","[1] Transition from LiveRepairReady to LiveRepair"hostname":","v":0,"name"ip-10-150-1-55.us-west-2.compute.internal":,""pidcrucible"",":level":475930}
,"time":"2023-09-22T23:14:58.702881383Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"msg"{:""msg":"[1] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active LiveRepair Active ds_transition to Faulted",[1] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) Active WaitQuorum New ds_transition to Active""v":,0","v"name":":crucible","0level":,"30name":"crucible","level":30,"time":"2023-09-22T23:14:58.702917482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.702921335Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal",2023-09-22T23:14:58.702936752Z"","pid"hostname":":4759ip-10-150-1-55.us-west-2.compute.internal","pid":}4759
}
{{"msg":""msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"[1] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active Faulted Active ds_transition to LiveRepairReady"crucible,"",v"":level"0:,"30name":"crucible","level":30{"msg":"Crucible stats registered with UUID: 430e0668-1443-47e0-a582-1e8e56ebed58",",,v""time:"0:",""name2023-09-22T23:14:58.702978681Z"":",crucible""time",hostname""level:""::"30ip-10-150-1-55.us-west-2.compute.internal","pid"2023-09-22T23:14:58.702975869Z":4759,"}
hostname":"{ip-10-150-1-55.us-west-2.compute.internal"","msgpid":"":4759,"[1] Transition from Faulted to LiveRepairReady"time"},:""v":
2023-09-22T23:14:58.70300481Z0",",name"":"hostname":crucible""{,"level"ip-10-150-1-55.us-west-2.compute.internal:"30,"pid""msg":":4759}
,"[2] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) Active Active New ds_transition to WaitActivetime""{:","v":"2023-09-22T23:14:58.703033836Zmsg"",:""0hostname",:""Crucible 430e0668-1443-47e0-a582-1e8e56ebed58 has session id: 3ccf3c6d-266e-431a-8c1f-43b8c19cd637"name",ip-10-150-1-55.us-west-2.compute.internal""v,""pid:"0:,4759":name}"
:""{crucible"crucible""msg,"":level"":,"30level":30[1] f40c9366-ba51-49e8-a7ac-2e8d70c8171b (b7f69b5e-7fe1-4604-9c09-0d5b9217b341) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.703072723Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal",",pid""2023-09-22T23:14:58.703074176Z"time:"4759:","}2023-09-22T23:14:58.70308182Z
"hostname,"{":"hostname"":"msg":"ip-10-150-1-55.us-west-2.compute.internal",ip-10-150-1-55.us-west-2.compute.internal"",pid""pid[0] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) New New New ds_transition to WaitActive""::,4759"4759}v
"}:{0
","msg"name:"":"crucible{",[1] Transition from LiveRepairReady to LiveRepair""level,"":v""30:msg":"0,"name":"[2] Transition from New to WaitActive"crucible",,""v":level,"":time30"0:,""name":"crucible"2023-09-22T23:14:58.703127235Z",",","hostnametime""::""level":302023-09-22T23:14:58.703137458Zip-10-150-1-55.us-west-2.compute.internal"",,""pid":hostname"4759:"}
ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"msg":"{"[0] Transition from New to WaitActivemsg"":","v":0,"Write to Extent 0:2:9 under repairname"",:""v"crucible:,"0,"timename"":":"crucible","level"2023-09-22T23:14:58.703150139Z":40,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,""time,"":"level":30{2023-09-22T23:14:58.703176559Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"[2] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) Active Active WaitActive ds_transition to WaitQuorumtime{"":"",msg2023-09-22T23:14:58.703190102Z"":",""v"hostnameWrite 1:0:9 past extent under repair 0"":",:"0vip-10-150-1-55.us-west-2.compute.internal"":,0",",namepid""::"4759"crucible}"
,"name":"{level":crucible"40","msg"level":":30[0] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) WaitActive New New ds_transition to WaitQuorum,"",time"":v"":0,"2023-09-22T23:14:58.703230046Zname"":","crucible"hostname",:""level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,",time""{:"time":""2023-09-22T23:14:58.703245288Z"msg":,""2023-09-22T23:14:58.703234702Z"hostname"Write 1:1:9 past extent under repair 0:"",,"ip-10-150-1-55.us-west-2.compute.internal""v,"":pid0"hostname":,4759":"}name
":"ip-10-150-1-55.us-west-2.compute.internal"crucible{",,""msg"level:"":"40pid":[0] Transition from WaitActive to WaitQuorum"4759,"v":0},"
name,"":"time":crucible"","level"2023-09-22T23:14:58.703282454Z:"30{,"hostname":""msg"ip-10-150-1-55.us-west-2.compute.internal":,""pid":4759,"}
time":[2] Transition from WaitActive to WaitQuorum"{","v""2023-09-22T23:14:58.703296893Zmsg"",:"":hostname"0:IO Write 1004 on eur 0 Added deps 1"",,"ip-10-150-1-55.us-west-2.compute.internal""v":,0",pid""name:"4759:"}name"crucible
":,{"""levelmsg""::40"crucible","level":30[0] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible,"","time":level"":302023-09-22T23:14:58.703335225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,",time"":time":""2023-09-22T23:14:58.703346689Z"2023-09-22T23:14:58.703338449Z","hostname",:""hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid"":,4759"pid"}:
4759}{
"msg":"[0] Transition from WaitQuorum to Active"{,"v":0,"name":""msg":"crucible","level":30[2] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.703394876Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.703404221Z"{",msg""":"msg":[1] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Active New New ds_transition to WaitActive","v":"0,"name":"crucible","Crucible stats registered with UUID: 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9"level":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.703437291Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid":2023-09-22T23:14:58.703441887Z"4759,"}
hostname":"{hostnameip-10-150-1-55.us-west-2.compute.internal"","pid":4759}
{"msg":""msg":Crucible 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 has session id: e97f0585-b400-40dd-918c-c5075104b8e6"","v":0,"name":"crucible","level":ip-10-150-1-55.us-west-2.compute.internal30","pid":4759}
:{""msg":"[1] Transition from New to WaitActive","time"[2] Transition from WaitQuorum to Active:"",,2023-09-22T23:14:58.703489126Z""v,"hostname":"ip-10-150-1-55.us-west-2.compute.internal"",:"0pid",:"4759name":"}
{"msg":"crucible","level":30"[0] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) New New New ds_transition to WaitActive","vv""::0,0"name":,","crucible"time":,""level":302023-09-22T23:14:58.703526492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",""pid"name"::","crucible"time,":"2023-09-22T23:14:58.703541225Z"","level"hostname:"30:"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.703558902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,}"
time{"msg":"[1] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.703609186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}"
:"{2023-09-22T23:14:58.70357726Z"","msg":"hostname":"[1] Transition from WaitActive to WaitQuorum"ip-10-150-1-55.us-west-2.compute.internal,"",v":"test live_repair::repair_test::test_live_repair_span_write_write ... pid"ok:4759
0,"name":"crucible","level":30}
{,"time":"47592023-09-22T23:14:58.703670629Z""msg":,}""
[0] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) WaitActive New New ds_transition to WaitQuorum{"msg":"c8868e06-7e14-4a37-b718-8f66383d6b07 is now active with session: 8778efe1-ed15-438a-a5de-27252f5bb5a3","v":0,"name":"crucible","level":30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.703724015Z","hostname":",ip-10-150-1-55.us-west-2.compute.internal"",time":"2023-09-22T23:14:58.703734906Z","hostname":""pid":4759ip-10-150-1-55.us-west-2.compute.internal","pid":hostname4759}"
}{"msg":"[1] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30
{,:""time"":ip-10-150-1-55.us-west-2.compute.internal""msg,""pid"::4759}
{"msg":"[1] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30"[0] Transition from WaitActive to WaitQuorum","v":0,",name"":time":2023-09-22T23:14:58.703797189Z""crucible"2023-09-22T23:14:58.703844345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"}
level":30{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30",,""time":,""hostname"time":"2023-09-22T23:14:58.703874427Zip-10-150-1-55.us-west-2.compute.internal","pid":4759","}hostname"
:"ip-10-150-1-55.us-west-2.compute.internal{","pid":4759"}msg"
{"msg":":"[0] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) WaitQuorum New New ds_transition to Active2023-09-22T23:14:58.703883695Z"",",v""hostname::"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":300,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.703997638Z"",,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","time"pid:":2023-09-22T23:14:58.704005004Z","hostname":""ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pidpid":":47594759}
}
{{"msg":"{"msg[2] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Active Active New ds_transition to WaitActive"":","v":0,"name":Crucible stats registered with UUID: a5309061-73a6-4f0b-bc10-fb7ff5b5432a""crucible",",v"":level"0:,"30name":"crucible","level":30"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,","time"time:"":"2023-09-22T23:14:58.704078113Z"2023-09-22T23:14:58.70408087Z",","hostname"hostname:":"ip-10-150-1-55.us-west-2.compute.internal","pid":"4759}
ip-10-150-1-55.us-west-2.compute.internal",,"{""pid"":msg4759":}"
time:Crucible a5309061-73a6-4f0b-bc10-fb7ff5b5432a has session id: 83187140-217c-405d-a29e-e92b0fb42bdc{"4759"",msg""v:"":}:[2] Transition from New to WaitActive
"","{v":0,"name":"crucible","level":30"msg":"[1] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) Active Faulted Active ds_transition to LiveRepairReady","time",:""v":2023-09-22T23:14:58.704144548Z"0,",hostname":""name":"ip-10-150-1-55.us-west-2.compute.internal","cruciblepid":"4759,"}level"2023-09-22T23:14:58.704093084Z
"{,""hostname":"msg":"ip-10-150-1-55.us-west-2.compute.internal"[2] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Active Active WaitActive ds_transition to WaitQuorum",":,"pidv""::047590},
30{",msg""name":":"crucible","level":30[1] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.704211504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{","msg"time:"":"[2] Transition from WaitActive to WaitQuorum2023-09-22T23:14:58.704221472Z"",""vname"",:""crucible"hostname",,"":time"level"":30ip-10-150-1-55.us-west-2.compute.internal","pid"::47590,"}name
:","2023-09-22T23:14:58.704231398Z"time":"{,2023-09-22T23:14:58.704263518Z""",:hostname""":"cruciblehostname"":"ip-10-150-1-55.us-west-2.compute.internalmsg":"","pid":[1] Transition from New to WaitActive"4759,"v":},0","name"level":":crucible"30,"level":30
","time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.704313034Z",","pid":hostname":4759"{}
"ip-10-150-1-55.us-west-2.compute.internal",,"{pid":"4759"msg}"
:"time":"{2023-09-22T23:14:58.704318962Z""msg":","[0] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) New New New ds_transition to WaitActive"hostname":","v"msgip-10-150-1-55.us-west-2.compute.internal"[2] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Active Active WaitQuorum ds_transition to Active"":,""[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30pid":4759,"}time"
,{:0","msg":name"":"crucible","level":30[1] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30"v":,"0time":,""name":"2023-09-22T23:14:58.704419396Z"crucible",,""level"hostname:":30"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":":{,""2023-09-22T23:14:58.704427779Ztime""msg:"":"2023-09-22T23:14:58.704371708Z""[0] Transition from New to WaitActive,","hostname"hostname":":",ip-10-150-1-55.us-west-2.compute.internal","pid":4759"v":}0
"{"ip-10-150-1-55.us-west-2.compute.internal"msg":","pid":[1] Transition from WaitActive to WaitQuorum"4759,,""}name"
:"vcrucible","level":30{"msg":","time":"[1] c8868e06-7e14-4a37-b718-8f66383d6b07 (8778efe1-ed15-438a-a5de-27252f5bb5a3) Active LiveRepairReady Active ds_transition to LiveRepair"2023-09-22T23:14:58.704537842Z",",hostname"":"v":ip-10-150-1-55.us-west-2.compute.internal0",,""pidname":":"4759crucible"}"
:{0,"name":""2023-09-22T23:14:58.704439767Zcruciblemsg"":"",","hostnamelevel":""[0] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) WaitActive New New ds_transition to WaitQuorum","v":0:,"30name"ip-10-150-1-55.us-west-2.compute.internal:"","crucible"pid",:"4759level":30}
{"msg":",,""time[2] Transition from WaitQuorum to Active"":,level,""""2023-09-22T23:14:58.70459067Z":,"30timehostname":""ip-10-150-1-55.us-west-2.compute.internal":,""pid":47592023-09-22T23:14:58.704587496Z"}v
",:"0{hostname":,"""nameip-10-150-1-55.us-west-2.compute.internal","pid"":,"time":"2023-09-22T23:14:58.70461357Z","hostname":"msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":[0] Transition from WaitActive to WaitQuorum"4759,"v":}0
,"name":"crucible","level":30{4759}
:"crucible",","level"time"::"302023-09-22T23:14:58.704657697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.704671648Z","hostname"":msg"":{"ip-10-150-1-55.us-west-2.compute.internal","pid":4759"}msg"
[0] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) WaitQuorum New New ds_transition to Active":",{"v":0,""msg"name:"":"[1] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) Active WaitQuorum New ds_transition to Active"crucible",",v":430e0668-1443-47e0-a582-1e8e56ebed58 is now active with session: 4b1819be-ee7b-4367-994d-70218dcb0427","v":0,"name":"crucible","level":30,"time":""2023-09-22T23:14:58.704709676Z"0msg",":""[1] Transition from LiveRepairReady to LiveRepair"level":30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.704731894Zname"",:""hostname"crucible:"","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.704738535Z"","hostname":","time":"2023-09-22T23:14:58.704753666Zip-10-150-1-55.us-west-2.compute.internal"",",pid""hostname"::"4759ip-10-150-1-55.us-west-2.compute.internal"},"
pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level"msg":":30[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.704796977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{",,"msg"time"hostname:"":":"2023-09-22T23:14:58.704803918Z"","ip-10-150-1-55.us-west-2.compute.internal"hostname":,""pid":4759[2] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) Active Active New ds_transition to WaitActiveip-10-150-1-55.us-west-2.compute.internal}"
,""pid":,"4759{v"}:
"0msg",":{"name":"crucible"",msg"":level":[1] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Active Active Active ds_transition to Faulted"30,"v":0,"name":"crucible","level":30","time":"2023-09-22T23:14:58.704851109Z","hostname":",[1] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) Active New New ds_transition to WaitActive""time":"ip-10-150-1-55.us-west-2.compute.internal,2023-09-22T23:14:58.704854183Z""",,"pid":4759}
v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.70488915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg"{:"[2] Transition from New to WaitActive",""v"msg:":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":300,"name":"crucible","level",:"30time,"time":"2023-09-22T23:14:58.70494225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) Active Active WaitActive ds_transition to WaitQuorum","v":"0,"name":""crucible"hostname",:"level"::30"2023-09-22T23:14:58.704928826Z","hostname":""ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid"":,"time4759":,""pid":4759}
test live_repair::repair_test::test_live_repair_update ... 2023-09-22T23:14:58.70498661Z{","}hostname":"
"ip-10-150-1-55.us-west-2.compute.internal"msg"ok:"
[1] Transition from Active to Faulted","{pid",":v":47590","}name"
{:"crucible","level"":msg":"30[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30msg":","time":"2023-09-22T23:14:58.705093074Z",",hostname"":"time":"ip-10-150-1-55.us-west-2.compute.internal[1] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) Active WaitActive New ds_transition to WaitQuorum2023-09-22T23:14:58.705088542Z""",","pid":,4759"v":0,"name":"crucible","level":30}
{,"time":""msg":"2023-09-22T23:14:58.70512194Z"hostname,"":"hostname":"[2] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) Active Active WaitQuorum ds_transition to Active"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",",pid":"4759pid",}"
v":0,"name":"crucible","{level":30"msg":"[1] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Active Faulted Active ds_transition to LiveRepairReady",",time":""2023-09-22T23:14:58.705172155Z"v",":hostname":"0,"ip-10-150-1-55.us-west-2.compute.internal"name":,"pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.705208421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":""crucible"5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 is now active with session: a14811af-cf7f-465a-acfb-10f5efafbe32":,,""vlevel""::04759,"30name":"}crucible"
,"level":30{"msg",":"[1] Transition from WaitActive to WaitQuorum"time":","v":2023-09-22T23:14:58.705255734Z"0,,""name":hostname"":crucible"","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time"[1] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) Active Active Active ds_transition to Faulted":","v":,2023-09-22T23:14:58.705247973Z0","name":"crucible","level":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":""2023-09-22T23:14:58.705306393Z","time{hostname"":":""msgip-10-150-1-55.us-west-2.compute.internal"":","2023-09-22T23:14:58.705281157Z"pid":[1] Transition from Faulted to LiveRepairReady"4759,,}"
{v":0,"name":"crucible","level":30,""msgtime""::""2023-09-22T23:14:58.705351624Z"[1] Transition from Active to Faulted",","vhostname""::"0,"name":"ip-10-150-1-55.us-west-2.compute.internalcrucible"",,""pid"level:":475930}
"{hostname"":msg",:""time"":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.705375362Z",,""pid":hostname":[1] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Active LiveRepairReady Active ds_transition to LiveRepair""4759,"vip-10-150-1-55.us-west-2.compute.internal"",}
"pid":4759}
:0,"name":{"crucible",""msg"level:"":30[1] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible",,""time"level:"":302023-09-22T23:14:58.705416667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",",pid"":time"4759:"}
2023-09-22T23:14:58.705431042Z{"{"msg":""msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":Crucible stats registered with UUID: 65642e3b-b3b8-4cfd-95ff-31b56650ceaf""crucible",",v"":level":030,"name":"crucible","level":30,"hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:14:58.705469656Z"pid",:"4759hostname":"}
{,""{time"msg"ip-10-150-1-55.us-west-2.compute.internal"":"msg",:[1] Transition from Faulted to LiveRepairReady":"2023-09-22T23:14:58.705474637Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":[1] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) Active WaitQuorum New ds_transition to Active"4759}
,,""v"v"::0{0,","namename"":""msgcrucible"::"""pid"crucible":Crucible 65642e3b-b3b8-4cfd-95ff-31b56650ceaf has session id: e2d26d29-154e-43c6-a019-1f3dd806ff40"4759,",level}""v":0,":name30":"crucible","level"
:30"{,"time":",""2023-09-22T23:14:58.705559601Z"msg",:""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[0] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Active LiveRepair Active ds_transition to Faulted"{,,""msglevel"":":time":"302023-09-22T23:14:58.705571822Z","hostname":"[1] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) Active LiveRepairReady Active ds_transition to LiveRepair","v":ip-10-150-1-55.us-west-2.compute.internal0","name":"crucible","level",":pid30":4759}
,"time":","2023-09-22T23:14:58.705596208Z"time{"":"msg":,"2023-09-22T23:14:58.705608467Z""v,"[0] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) New New New ds_transition to WaitActive":"0,hostname,":""name":"crucible","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,""v""":msg0",:""name":[1] Transition from LiveRepairReady to LiveRepair"",crucible""v,":"0level,time""name":"crucible","level":30":30,"time":"2023-09-22T23:14:58.705679478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{:""2023-09-22T23:14:58.705641117Zmsg"":",","Write to Extent 0:0:9 under repair"time",":v""2023-09-22T23:14:58.705688902Z:"0,",name":"hostnamecrucible"hostname:""",":level"":40ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid":pid":47594759}
},
{"msg":"[0] Transition from Active to Faulted{","v":0,"name":""crucible"msg",:""level":30[1] Transition from WaitQuorum to Active","v":0,"name":"crucible",""timelevel""::"302023-09-22T23:14:58.705729029Z",",time"":"hostname":"2023-09-22T23:14:58.705762234Z","ip-10-150-1-55.us-west-2.compute.internal"hostname",:""pid":4759}ip-10-150-1-55.us-west-2.compute.internal"
{,"time":""2023-09-22T23:14:58.705774278Z","hostname"hostname"::""ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"pid":}
4759"msg":"}
{Write to Extent 0:0:9 under repair",""msg"v{","pid""::msg"47590:,}"
name"":"crucible",[0] Transition from New to WaitActive"{level:""msg":"":40[2] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) Active Active New ds_transition to WaitActive"[2] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Faulted LiveRepair Active ds_transition to Faulted",",v"":v"0:,0,"name":",""timecrucible"name"":,":""level":2023-09-22T23:14:58.705850012Z"crucible"30,,""hostname":"level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"{","msg":v",""[1] client skip 3 in process jobs because fault","v":0,"timename""::""crucible",","leveltime":"2023-09-22T23:14:58.705876514Z""::,"030hostname":,""name":"ip-10-150-1-55.us-west-2.compute.internalcrucible","level":30","pid",:"4759time":"}2023-09-22T23:14:58.705875857Z2023-09-22T23:14:58.705903968Z
"",",hostname"":"hostname":"ip-10-150-1-55.us-west-2.compute.internal"{ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:":47594759,},""
"time"":{""2023-09-22T23:14:58.705912846Z"msg:"":","downstairs"hostname"[2] Transition from Active to Faulted}"
msg----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:6  downstairs:6
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1 AckReady  1000   Write       1  Done  Done  Done  false
    2 NotAcked  1001    Read       1  Done  Done  Done  false
{    3 NotAcked  1002  WriteU       1  Done  Done  Done  false
":,""    4 AckReady  1003   Write       1[2] Transition from New to WaitActive   New  Skip   New  false
"    5 NotAcked  1004    Read       1   New  Skip   New  false
:,""v":0,    6 NotAcked  1005  WriteU       1   Newv  Skip   New  false
   STATES      DS:0   DS:1   DS:2   TOTAL
"    New           6   msg"   3   :""   6   name"  15
    Sent          0      0      0      0
    Done          0      0      0      0
    Skipped       0      3      0      3
"    Error      :0   0   ,   0      0   [1] job 1000 middle: AckReady   0
Last Flush: "","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.706075878Z","hostname":"0 0 0 
Downstairs last five completed:ip-10-150-1-55.us-west-2.compute.internal","pid":4759name"}:"
crucible","level":30{"msg":":
[0] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible,"","time"level"::Upstairs last five completed:  30
","time":"2023-09-22T23:14:58.70614485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30"crucible","level":2023-09-22T23:14:58.706120864Z"30,"hostname":,""ip-10-150-1-55.us-west-2.compute.internal","pid":4759time":"}
2023-09-22T23:14:58.706193745Z{","hostname":",msg""":"ip-10-150-1-55.us-west-2.compute.internal"time,Write to Extent 0:0:9 under repair""pid,"""::4759v"ip-10-150-1-55.us-west-2.compute.internal:"}0,,""
"pidname""::"4759crucible",{2023-09-22T23:14:58.706205336Z"","":"level",:downstairs40"""}msg
"hostname":":{""ip-10-150-1-55.us-west-2.compute.internal"msg":[0] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) WaitQuorum New New ds_transition to Active",",",[1] changed 3 jobs to fault skipped""",time""v:"":v":2023-09-22T23:14:58.706256455Z0"0pid,,,""name"hostname""::""crucible"name,ip-10-150-1-55.us-west-2.compute.internal""level,"":pid30":":""crucible"4759,":level"4759}:
}30
{,"time":""msg":"2023-09-22T23:14:58.706296125Z","Write to Extent 0:0:9 under repair"hostname{",:""v":,"0ip-10-150-1-55.us-west-2.compute.internal,time":""msgname"",:"""pid""crucible:"4759,",2023-09-22T23:14:58.706308232Z":"level"""::40","downstairshostname":""[2] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) Active Active WaitActive ds_transition to WaitQuorum}ip-10-150-1-55.us-west-2.compute.internal"","
,",pid":"4759{v"time""msg:""::}2023-09-22T23:14:58.706337967Z""0
,","hostnamename"{:""[1] 5ef2dbd8-14fa-49c7-9429-be5eaaf14af9 (a14811af-cf7f-465a-acfb-10f5efafbe32) Active LiveRepair Active ds_transition to Faulted:"",""msg":ip-10-150-1-55.us-west-2.compute.internalv"",:"0pid,"":crucible""4759name"[0] Transition from WaitQuorum to Active",}:
""crucible{,"""msglevelv":0,"",:"":"level[1] client skip 3 in process jobs because fault"":3030,name":""crucible"v",":level":030,"name":","crucibletime"":,""level":302023-09-22T23:14:58.706399349Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.70640666Z",",pid"":,4759"hostname":"time}"
:ip-10-150-1-55.us-west-2.compute.internal"{",2023-09-22T23:14:58.706410805Z""msg,"":,"hostname"":[1] Transition from LiveRepair to Faulted"""timepid"ip-10-150-1-55.us-west-2.compute.internal,"",v"":pid0",:"4759name"":,4759":"":crucible"":,downstairs""level}"
:}"{30
"2023-09-22T23:14:58.706399565Z"msg"{,:"""msg",[1] job 1000 middle: AckReady""time",:""v"::02023-09-22T23:14:58.706463033Z,"",hostname"name"""hostname:"":"crucible":",ip-10-150-1-55.us-west-2.compute.internal""level,"":pid30[1] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) Active New New ds_transition to WaitActive"ip-10-150-1-55.us-west-2.compute.internal",":"v"4759:0,,}"
time,""name"{"::""msg""pid"crucible"2023-09-22T23:14:58.706491382Z:"",,":level":Extent 0 Aborting repair""4759,hostname""v:"":30}0ip-10-150-1-55.us-west-2.compute.internal,"","namepid""::"4759crucible",,""":"level"downstairs:"40}

,","{time":"{"timemsg""::""2023-09-22T23:14:58.706530551Z"2023-09-22T23:14:58.706543062Z"",,"[1] notify = true for 1001"hostname,"""v:"":hostname"0ip-10-150-1-55.us-west-2.compute.internal,":,""namepid""::"4759"crucible}"
ip-10-150-1-55.us-west-2.compute.internal",,"msg"pid"level":"::304759"}
[2] Transition from WaitActive to WaitQuorum"{,""v"msg":":0[1] Transition from New to WaitActive",,",""vname"time"":0:,":""name":"crucible2023-09-22T23:14:58.706591383Z"crucible",,"""level"hostname",":30:"level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759,",":""time":"downstairs"2023-09-22T23:14:58.706622298Z"},"
hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
",msg{""":"time"msg":"[1] notify = true for 1002":","v":02023-09-22T23:14:58.706625361Z","name[1] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) Active WaitActive New ds_transition to WaitQuorum",",":""hostname"crucible":,v":""0level","ip-10-150-1-55.us-west-2.compute.internalname""::30,""crucible"pid,"":level"4759:30}
,"time":"2023-09-22T23:14:58.706675778Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":",""downstairstime":"msg"":2023-09-22T23:14:58.706682916Z"}"
,"hostname":"{[2] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) Active Active WaitQuorum ds_transition to Active"",msg"":"v":[1] changed 3 jobs to fault skipped"ip-10-150-1-55.us-west-2.compute.internal,0"",v,""pid":0","name"name":":crucible":"4759,"crucible"}level"
:,"30level":30{"msg":","time":"[1] Transition from WaitActive to WaitQuorum"2023-09-22T23:14:58.706742472Z",",hostname"":"v":0ip-10-150-1-55.us-west-2.compute.internal",,""pid":name":"4759crucible",",",":"time""downstairs:"level""}
2023-09-22T23:14:58.706744429Z":30,"hostname":"{ip-10-150-1-55.us-west-2.compute.internal"",msg":""pid":4759}
[1] 430e0668-1443-47e0-a582-1e8e56ebed58 (4b1819be-ee7b-4367-994d-70218dcb0427) Faulted LiveRepair Faulted ds_transition to Faulted","v":,0","time":name"{":"crucible"2023-09-22T23:14:58.706777199Z",""levelmsg":",":"30[2] Transition from WaitQuorum to Active"hostname":","v":0ip-10-150-1-55.us-west-2.compute.internal",",name":""pid",:crucible""4759,time":""}level2023-09-22T23:14:58.706809618Z"
,"":30hostname"{:""ip-10-150-1-55.us-west-2.compute.internalmsg"":","pid":4759}
[1] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) Active WaitQuorum New ds_transition to Active","v":0test live_repair::repair_test::test_repair_abort_basic ... {"okmsg
":",,"[1] Transition from LiveRepair to Faulted"time"":,name":""crucible"v",":"0level",2023-09-22T23:14:58.706833737Z"":name":30","crucible"hostname":","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.706880196Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759,}"
"msg"time":":{"2023-09-22T23:14:58.706885928Z"","msg":"hostname":a5309061-73a6-4f0b-bc10-fb7ff5b5432a is now active with session: 16f3c8a3-5547-4c93-b43b-aa71fb92fbd0""[1] Transition from WaitQuorum to Active",",ip-10-150-1-55.us-west-2.compute.internal"v",v":"0:","pid":0name"4759,":"name"crucible}:
""crucible,""{level":,"30msg":""Extent 0 Aborting repair"level",":v":300,"name":"crucible","level":40,"time":"2023-09-22T23:14:58.706943587Z","hostname":","time":"2023-09-22T23:14:58.70695209Z"ip-10-150-1-55.us-west-2.compute.internal",",hostname"":",pid"ip-10-150-1-55.us-west-2.compute.internal":","4759pid":time4759"}:}

{""msg":"2023-09-22T23:14:58.706949964Z"{Abort repair on extent 0: All downstairs are Faulted",","v":0hostname",""name":"msg"crucible":":","level":ip-10-150-1-55.us-west-2.compute.internal"50,"pid":4759[2] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) Active Active New ds_transition to WaitActive"},"
v{"msg":"[1] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30":0,"name":"crucible","level",:"30time":"2023-09-22T23:14:58.707039436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"}
time":"2023-09-22T23:14:58.707003904Z","hostname{,":""time":""msg"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.707059504Z",":","pid":hostname4759[1] Transition from Active to Faulted"","}:"v":
0ip-10-150-1-55.us-west-2.compute.internal",",name"":"pid"crucible:"4759,"level":}30
{"msg":"[2] Transition from New to WaitActive","v":0,"name":,""crucible"time",:""level":2023-09-22T23:14:58.707106641Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,{"time":""2023-09-22T23:14:58.707130206Z"msg":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) Active Faulted Active ds_transition to LiveRepairReady"}
,"v":0,{"name":""crucible"msg",":level":"30[2] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.707168857Z","hostname":","ip-10-150-1-55.us-west-2.compute.internaltime"",":pid"":47592023-09-22T23:14:58.707180231Z"},
"hostname":"ip-10-150-1-55.us-west-2.compute.internal","{pid":4759}
"msg":"{[1] Transition from Faulted to LiveRepairReady","v":"0msg,""name"::""crucible","[2] Transition from WaitActive to WaitQuorum"level":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.70722831Z",","time"hostname:":""2023-09-22T23:14:58.707233459Zip-10-150-1-55.us-west-2.compute.internal"",","pid"hostname:"4759:"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}{
"msg":"{"msg":"[1] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"[2] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) Active Active WaitQuorum ds_transition to Active"crucible",",v":"0level",":name":30"{crucible","level":30"msg":"Crucible stats registered with UUID: 9cf1cfee-1bb8-4140-8e2f-38535cb4be54","v":0,"name":"crucible",",level":",time"test live_repair::repair_test::test_repair_abort_all_failed_reserved_jobs ... 30timeok
":"":"2023-09-22T23:14:58.707293995Z"2023-09-22T23:14:58.707297331Z",","hostname":hostname"",":"time":ip-10-150-1-55.us-west-2.compute.internal""ip-10-150-1-55.us-west-2.compute.internal",,""2023-09-22T23:14:58.707316526Z"pid"pid":,4759":hostname"4759:"}}
ip-10-150-1-55.us-west-2.compute.internal"
{,"pid":4759{"}msg
"":msg"":"{[1] Transition from LiveRepairReady to LiveRepair"[2] Transition from WaitQuorum to Active"",msg,":"""v":v"0:Crucible 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 has session id: b5809739-3766-4449-8296-1dc8c62282b90,"",",name"":"v":crucible0name":","","name":"levelcrucible"",":crucible"level30",":level":3030,"time":"2023-09-22T23:14:58.70741212Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal",,2023-09-22T23:14:58.707415484Z"""pid"time",":hostname4759:""}:2023-09-22T23:14:58.707415882Z""
,ip-10-150-1-55.us-west-2.compute.internal"","hostname{"pid"::4759""msg"}:ip-10-150-1-55.us-west-2.compute.internal"
,""pid":4759}{
"65642e3b-b3b8-4cfd-95ff-31b56650ceaf is now active with session: eda3e3b2-1531-45ea-b904-d701ad430ba2"msg":","v":0{,"name":"crucible",[0] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) New New New ds_transition to WaitActive"""levelmsg"","::"v":30Write to Extent 0:0:9 under repair"0,,""v"name"::"0crucible",",name":""level"crucible":,"30level":40,"time":"2023-09-22T23:14:58.707491616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.707505987Z","{,time":"""hostname2023-09-22T23:14:58.707507135Z"msg"","::""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"ip-10-150-1-55.us-west-2.compute.internal":,"4759pid":[1] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) Active Active Active ds_transition to Faulted4759"}
,}"
{v":0,"name":""crucible"msg{",":"level":"30msg":"[0] Transition from New to WaitActive","Write to Extent 0:0:9 under repair"v":,"0v,":"0name",:""name":crucible""crucible",","levellevel"":,:3040"time":"2023-09-22T23:14:58.707570908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":","{2023-09-22T23:14:58.707591418Z"time":,"""hostname"msg":":"2023-09-22T23:14:58.707592067Z"ip-10-150-1-55.us-west-2.compute.internal",[1] Transition from Active to Faulted",",pid":"4759"v":hostname0},"
:""name":"crucible"{ip-10-150-1-55.us-west-2.compute.internal",,"""level"msg":"pid"::475930}
[0] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) WaitActive New New ds_transition to WaitQuorum","v":0,"name":{"crucible","level":"30msg":","[1] client skip 3 in process jobs because fault"time":,""v":02023-09-22T23:14:58.707648889Z","name",":"hostname":crucible"","level":30ip-10-150-1-55.us-west-2.compute.internal","pid,"":time"4759:"}
2023-09-22T23:14:58.707663103Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal",""pid"msg"::4759","}time
":"[1] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) Active Faulted Active ds_transition to LiveRepairReady",2023-09-22T23:14:58.707677823Z""{,v":"0"hostname":,""name":"msg"ip-10-150-1-55.us-west-2.compute.internalcrucible:"","","pid[0] Transition from WaitActive to WaitQuorum"level"",::30"4759v":0,",":""downstairs"name":}"
crucible","level":30,"time{":"2023-09-22T23:14:58.707729835Z"","msg"hostname":":"[1] job 1000 middle: AckReady"ip-10-150-1-55.us-west-2.compute.internal",,"","vtime""pid"::0:,""4759name":"2023-09-22T23:14:58.707745732Z}crucible"
",",level"":hostname"{:30""msg"ip-10-150-1-55.us-west-2.compute.internal":","pid":4759[1] Transition from Faulted to LiveRepairReady"}
,"v":0,"{name":"crucible"","msg"level":":,30"time":"2023-09-22T23:14:58.707782816Z"[0] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) WaitQuorum New New ds_transition to Active",,""v":hostname0",":"name":"crucible","ip-10-150-1-55.us-west-2.compute.internallevel"":,"30pid":,4759"time":,""":"downstairs"2023-09-22T23:14:58.707803352Z"}
,"hostname":","ip-10-150-1-55.us-west-2.compute.internaltime"{,""pid":"":msg"2023-09-22T23:14:58.707818875Z"4759:,""hostname"}:[1] changed 3 jobs to fault skipped"
","v":0ip-10-150-1-55.us-west-2.compute.internal,"{",name"":pid":"4759msg":""crucible"},
[1] 65642e3b-b3b8-4cfd-95ff-31b56650ceaf (eda3e3b2-1531-45ea-b904-d701ad430ba2) Active LiveRepairReady Active ds_transition to LiveRepair""level",":{v"30:0","msgname":"":"crucible","level":[0] Transition from WaitQuorum to Active30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.707877398Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time",:""pid":47592023-09-22T23:14:58.70788547Z",,","time"hostname":":""":2023-09-22T23:14:58.707892113Z""ip-10-150-1-55.us-west-2.compute.internal","downstairshostname",:"""}pid"ip-10-150-1-55.us-west-2.compute.internal":
4759,"pid":}4759
{}
{"msg":""{msg":""[1] Transition from LiveRepairReady to LiveRepair"[1] a5309061-73a6-4f0b-bc10-fb7ff5b5432a (16f3c8a3-5547-4c93-b43b-aa71fb92fbd0) Active LiveRepair Active ds_transition to Faulted"msg",:"","v"v"::00,,"[1] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) Active New New ds_transition to WaitActive""name":,"name"vcrucible"":""crucible",",:"0level"level":,30":name"30:"crucible","level":30,"time":"2023-09-22T23:14:58.707980641Z",","time"hostname",":"time":":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.707985939Z""2023-09-22T23:14:58.707982707Z,","pid"":,4759hostname":"}"
hostname"ip-10-150-1-55.us-west-2.compute.internal":","{pid":ip-10-150-1-55.us-west-2.compute.internal"4759,""{msg":"pid"}[1] 0 final dependency list []""msg,"v":0,"name":"crucible","level"":":Crucible stats registered with UUID: 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf"30,"v":0,"name":"crucible","level":30:4759}
,,"
time{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30":"2023-09-22T23:14:58.708053678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":"Crucible 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf has session id: 39db2329-7823-4e21-9a0a-6f2899f6b59a","v":0,"name":"crucible","level":30"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible",,","time"level:"":""time30"time":"2023-09-22T23:14:58.708047756Z":","2023-09-22T23:14:58.708166404Z"hostname":,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid"":,"pid4759":47592023-09-22T23:14:58.708100201Z}"
,{"msg":"[0] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30"time":"2023-09-22T23:14:58.708191306Z",,""hostname":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"ip-10-150-1-55.us-west-2.compute.internal":,"4759pid":,"4759time":"}
2023-09-22T23:14:58.708245409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759}
}
{{""msg":"msg"Extent 0 Aborting repair":","v":0,"name":"crucible","level":40[1] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":""2023-09-22T23:14:58.708302662Zmsg"":","hostname":"[0] Transition from New to WaitActive","v":ip-10-150-1-55.us-west-2.compute.internal"0,,""pid",name","time":"":"2023-09-22T23:14:58.70831255Z"downstairs","hostname"}:
"ip-10-150-1-55.us-west-2.compute.internal","{pid":4759"}msg
":"":"crucible","[1] 0 final dependency list []"level":{,30"v":0,"name":""msg"crucible":","level":30[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level"::30,4759"time":"}
,2023-09-22T23:14:58.708359747Z""time",:""hostname"{2023-09-22T23:14:58.708366846Z:"",""hostnameip-10-150-1-55.us-west-2.compute.internal":"","pid":4759}
{"msg":"[0] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) WaitActive New New ds_transition to WaitQuorum","v":0,",name"":"time"crucible:"","level":2023-09-22T23:14:58.708374394Z"30ip-10-150-1-55.us-west-2.compute.internal",,""hostname":"pid":4759,ip-10-150-1-55.us-west-2.compute.internal"",":""downstairs","}time":"2023-09-22T23:14:58.708417927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
msg":"{Extent 0 Create and send noop jobs",""v":msg0":"[0] Transition from WaitActive to WaitQuorum","v":0,"
name":"crucible,{"","namelevel"":":crucible","level":40pid":4759}
,"time":"2023-09-22T23:14:58.708465961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","{pid":4759}
30"msg":"{"msg":"[1] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) Active WaitQuorum New ds_transition to Active"Abort repair on extent 0 starting with id 1003 deps:[JobId(1002), JobId(1001), JobId(1000)]",",v":"0v",:"0,,"name":"crucible","level":30"msg":"[1] 0 final dependency list []","v":0,"name":"crucible","level":30"time":","2023-09-22T23:14:58.70849167Z"time":","hostname":2023-09-22T23:14:58.708516368Z"",ip-10-150-1-55.us-west-2.compute.internal"name":"crucible","time"level:"":402023-09-22T23:14:58.708528483Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"},"
time":"2023-09-22T23:14:58.708547976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"",,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"{pid":4759}
{"msg":":4759}
[0] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level"":30msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.708628077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,{"time":""msg"2023-09-22T23:14:58.708638978Z":","hostname"[0] Transition from WaitQuorum to Active":","v":0,"ip-10-150-1-55.us-west-2.compute.internal"name",":"pid":crucible4759","level"}:
30{"msg":"[2] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) Active Active New ds_transition to WaitActive","v":,0","time"name":":"crucible","2023-09-22T23:14:58.708688174Zlevel":"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":"2023-09-22T23:14:58.708712969Z","hostname"[1] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) Active New New ds_transition to WaitActive:"","v":ip-10-150-1-55.us-west-2.compute.internal"0,","pid"name"::"4759crucible","}level
":30{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.708752662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.708766001Z",""hostnamemsg":"":"[1] Transition from New to WaitActive","ip-10-150-1-55.us-west-2.compute.internalv":0,"name":"crucible","level":30","pid":4759}
,"time":"2023-09-22T23:14:58.70880214Z","hostname{":"ip-10-150-1-55.us-west-2.compute.internal",""msg"pid"::test live_repair::repair_test::test_repair_dep_cleanup_done ... 4759ok
}
test live_repair::repair_test::test_repair_abort_reserved_jobs ... ok
{"msg":"[1] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible"","level":30[2] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.708888933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum,""time",:""v":0,2023-09-22T23:14:58.70890282Z""name":,""crucible"hostname","level":30,"time":"2023-09-22T23:14:58.7089672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30:"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.709042813Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":"4759msg":"}
{"msg":"[1] Transition from WaitQuorum to Active","v":0[2] Transition from WaitActive to WaitQuorum,""name":"crucible,"","v"level":30,"time":"2023-09-22T23:14:58.709124206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) Active Active New ds_transition to WaitActive","v":0,"name":"crucible:","0level":30,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.70917117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{",msg"":"time":"[2] Transition from New to WaitActive"2023-09-22T23:14:58.709177679Z","v":,0","hostname"name:"":"crucible",ip-10-150-1-55.us-west-2.compute.internal"","level"pid":30:4759},"
time":"2023-09-22T23:14:58.709221182Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759"}
{"msg":"msg":"[2] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name[2] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) Active Active WaitQuorum ds_transition to Active"":"crucible",,""v"level:":030,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.709281541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,}{
"msg":"Crucible stats registered with UUID: 8ee74e9c-4380-4bf2-913b-9825b6e2923a{",""vmsg":"{[2] Transition from WaitActive to WaitQuorum","v":0","msg":name"":"crucible"Crucible stats registered with UUID: 4dc045ec-5a1e-410e-8896-79fc1c563313","level",:30"v":0,""name":"time"crucible:"","level":2023-09-22T23:14:58.709286218Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal",,""pid":time":"47592023-09-22T23:14:58.709345001Z"}
,,""hostname"time"::""2023-09-22T23:14:58.709354693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"""pid:"0,"name":"crucible",:"4759level":}30
,"pid":4759{}
,""msg":"time":"{2023-09-22T23:14:58.709394578Z","hostname":""[2] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) Active Active WaitQuorum ds_transition to Activemsg""ip-10-150-1-55.us-west-2.compute.internal",,"v":0,"name":"crucible",""pid"level:":4759{30}
"msg":"{[2] Transition from WaitQuorum to Active","v":0",msg"":name"":","crucible"time":,""Crucible 8ee74e9c-4380-4bf2-913b-9825b6e2923a has session id: 043467c1-af50-49df-8a85-89945b943bab"level2023-09-22T23:14:58.709438696Z,"","hostname":"vip-10-150-1-55.us-west-2.compute.internal":","pid":4759Crucible 4dc045ec-5a1e-410e-8896-79fc1c563313 has session id: edb2d2fc-df76-4605-ab5a-337da0dc2f65"}
{":0,"name":"crucible"""msg,""::level"30":30[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":","time":"2023-09-22T23:14:58.709498714Z","2023-09-22T23:14:58.709499848Z","timehostname""::""2023-09-22T23:14:58.709506438Z"ip-10-150-1-55.us-west-2.compute.internal",",,pid"":hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"4759:4759},}"
hostname":"ip-10-150-1-55.us-west-2.compute.internal{","pid":"4759msg":""}v
"3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf is now active with session: 00bc7fd9-cb37-40a9-bc42-8026d7e44a56:"0,","v"name"::"crucible"0,,""level"name"
:"crucible","{level":30{"msg":""msg":"9cf1cfee-1bb8-4140-8e2f-38535cb4be54 is now active with session: 1c47f668-99d2-4cb8-b990-8ee01032ed06","v":0,"name":"crucible"[0] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) New New New ds_transition to WaitActive","level",":v"30,:"0time,":"2023-09-22T23:14:58.709590871Z","hostname":":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"","name"time"::""[1] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) Active Active Active ds_transition to Faulted"2023-09-22T23:14:58.709623599Z",","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,}"
time":"2023-09-22T23:14:58.709609868Z","hostname{":"ip-10-150-1-55.us-west-2.compute.internal"","msgpid":"":4759}
[0] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) New New New ds_transition to WaitActive","v":0,"name":"{crucible","level":30"msg":"v":0[1] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) Active Active Active ds_transition to Faulted",","name"v":","time":"crucible","2023-09-22T23:14:58.709689039Z"level":,"30cruciblehostname":"","level"ip-10-150-1-55.us-west-2.compute.internal":,"30pid"::47590}
,"name":"crucible"{,"level":,"30time":""msg":"2023-09-22T23:14:58.709711481Z"[0] Transition from New to WaitActive",","hostname":,""time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:14:58.709718829Z"pid":,4759,}
{"hostname":"ip-10-150-1-55.us-west-2.compute.internal"","time"pid:"":2023-09-22T23:14:58.709730037Z"4759,"}hostname"
:"ip-10-150-1-55.us-west-2.compute.internal","pid":4759{}
""msgmsg""::"{"[0] Transition from New to WaitActive","v"":msg"0:","[1] Transition from Active to Faulted"[1] Transition from Active to Faulted,"vv""::"00,,,""v":name0"":name":"crucible","level":"30crucible","level":name30":"crucible",,""name"level":":crucible"30,"level":30,"time":","time"2023-09-22T23:14:58.709825458Z","time":","hostname":2023-09-22T23:14:58.709839856Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal4759,""time":"}
:"2023-09-22T23:14:58.709829312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30{"msg":","time":,"[1] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) Active Faulted Active ds_transition to LiveRepairReady2023-09-22T23:14:58.709893684Z","hostname":""ip-10-150-1-55.us-west-2.compute.internal",","pid"v:":47590","pidname"":}:"
crucible{","level":302023-09-22T23:14:58.709838544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:14:58.709941421Z"",msg"":hostname"":"4759[0] Transition from WaitActive to WaitQuorum"ip-10-150-1-55.us-west-2.compute.internal",,"}v
":"0,"name"{:"crucible"pid","":msglevel4759"}
{":30:"[1] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.710003207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid",:"4759time":"}
2023-09-22T23:14:58.710015662Z"",msg":{""msg":"[0] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30{"msg"[1] Transition from Faulted to LiveRepairReady:"","time,"[0] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759v"}
:0,"name":"{crucible","level":"30msg":","time""[1] Transition from Faulted to LiveRepairReady":",:""2023-09-22T23:14:58.710049127Z"v":02023-09-22T23:14:58.710072115Z,,""name"hostname:"":"crucible"",","levelip-10-150-1-55.us-west-2.compute.internalhostname,"",:"""pid":4759}time"
:"{:30"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.710120838Z"","hostname":",2023-09-22T23:14:58.710091083Z"ip-10-150-1-55.us-west-2.compute.internaltime"",:""pid":"2023-09-22T23:14:58.710129478Z4759",}",hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
ip-10-150-1-55.us-west-2.compute.internal{""msg":,""pid":4759[1] 3f90871a-d6d4-40b8-9b9f-b0ce0b47ecdf (00bc7fd9-cb37-40a9-bc42-8026d7e44a56) Active LiveRepairReady Active ds_transition to LiveRepair"},
"v":0,"name":"crucible","level":30"{hostname":"ip-10-150-1-55.us-west-2.compute.internal",""msg"pid"::"4759}
,"{"msg":"[1] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30time":"
2023-09-22T23:14:58.710174595Z,""time",":{"hostname":"2023-09-22T23:14:58.710204778Z""msg",:""ip-10-150-1-55.us-west-2.compute.internal"hostname",":pid":4759}
{""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid[1] Transition from LiveRepairReady to LiveRepair"":,"4759v":0,"}name"
:"crucible","level":30{"msg":"[0] Transition from WaitActive to WaitQuorum[1] Transition from New to WaitActive"","v":0,"name",:"",v"crucible"",time":"2023-09-22T23:14:58.710281077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{[1] 9cf1cfee-1bb8-4140-8e2f-38535cb4be54 (1c47f668-99d2-4cb8-b990-8ee01032ed06) Active LiveRepairReady Active ds_transition to LiveRepair""msg":"Write to Extent 0:1:9 under repair",,""v":v"0:,0","name":"namecrucible"""level":30,"level":40::0"crucible",",,"","nametime":"crucible","level":30level":30":"2023-09-22T23:14:58.710326524Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:14:58.710349871Z",","time"hostname:":""pid":4759ip-10-150-1-55.us-west-2.compute.internal}
time{"msg":"[1] 1004 final dependency list [JobId(1001), JobId(1002), JobId(1003)]","v":0,"name":"crucible",""level":30:"2023-09-22T23:14:58.710321781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,"4759"time":"},
"2023-09-22T23:14:58.710408944Zpid":4759}
{","hostname":"{"msg""ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.710346056Z,msg":""[1] Transition from LiveRepairReady to LiveRepair",","v":hostname"0:","name":"crucibleip-10-150-1-55.us-west-2.compute.internal"",",pid"":level":475930}"
pid":4759,"":"downstairs"}{
"msg":":[0] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) WaitQuorum New New ds_transition to Active"","v":0,"name":"crucible","[1] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) Active WaitActive New ds_transition to WaitQuorumlevel":"30,"v":{0,"name":""crucible"msg,""level:"":30[1] 1005 final dependency list [JobId(1004), JobId(1001)]","v":0,"name":"crucible","level":30,"time":",2023-09-22T23:14:58.710454326Z"",time",,""hostnametime":"2023-09-22T23:14:58.710502192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
":"":"2023-09-22T23:14:58.710488939Z"ip-10-150-1-55.us-west-2.compute.internal",","hostname":"pid":4759ip-10-150-1-55.us-west-2.compute.internal"}
,{"msg":"Write to Extent 0:1:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:58.710570816Z","hostname":""pid":ip-10-150-1-55.us-west-2.compute.internal",4759"pid":4759}}

{{time"":"msg":""2023-09-22T23:14:58.710497958ZWrite to Extent 0:1:9 under repair""msg,,""v"":hostname0",:""name":":crucibleip-10-150-1-55.us-west-2.compute.internal"","","levelpid""::40[0] Transition from WaitQuorum to Active"4759,"}v"
:0,"name"{,":"time""crucible":msg"":,"2023-09-22T23:14:58.71062354Z"",level""[1] Transition from WaitActive to WaitQuorum":hostname,"":"v":300ip-10-150-1-55.us-west-2.compute.internal",","name"pid:"":4759crucible"},
"level":30{"msg":"Write to Extent 0:2:9 under repair","v":0,"name":"crucible","level":40,"time":",2023-09-22T23:14:58.710652745Z"",,""time"timehostname""::"":"2023-09-22T23:14:58.710661037Z"ip-10-150-1-55.us-west-2.compute.internal,"","hostname"pid:"":2023-09-22T23:14:58.710647197Z"4759ip-10-150-1-55.us-west-2.compute.internal","}pid
":,"{4759hostname":}"
"ip-10-150-1-55.us-west-2.compute.internal"{msg,"":msg"":""pid":4759Write 1:0:9 past extent under repair 0","}v[1] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) Active WaitQuorum New ds_transition to Active"":,0",v""name:"0:,""
namecrucible"":","crucible"level",:"40level":30{"msg":",,""time"time:"":"[1] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) Active New New ds_transition to WaitActive"2023-09-22T23:14:58.710715336Z"2023-09-22T23:14:58.710714507Z,"","hostname"hostname:"":","v":ip-10-150-1-55.us-west-2.compute.internal"0ip-10-150-1-55.us-west-2.compute.internal,,"","pidpid""::47594759"}}

{name":""{crucible""msg"msg:"":","level"IO Write 1013 on eur 0 Added deps 1[1] Transition from WaitQuorum to Active"",:,30""vv""::00,,""namename"test live_repair::repair_test::test_repair_dep_cleanup_sk_repair ... :"":crucible""okcrucible,"",level"":level30"
,:"40time":"2023-09-22T23:14:58.710760187Z",,""hostnametime"":":,2023-09-22T23:14:58.710777504Z""",time"":"hostname":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.710782976Z",,ip-10-150-1-55.us-west-2.compute.internal""",hostname""pid:"":pid"4759ip-10-150-1-55.us-west-2.compute.internal"},
":pid{"4759:"4759msg":}"
}
[2] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) Active Active New ds_transition to WaitActive{",""v"msg:"0:","name":"[1] 1007 final dependency list []"crucible"{,,""vlevel""::030,"name":""crucible"msg",:""level":30[1] Transition from New to WaitActive",",v"":time":"0,"name2023-09-22T23:14:58.710827682Z,"","time"hostname:"":""2023-09-22T23:14:58.710834781Z":",ip-10-150-1-55.us-west-2.compute.internalcrucible"""hostname,"":pid"",":ip-10-150-1-55.us-west-2.compute.internal4759"level,}"
pid":"{4759:",msg"""::""30downstairs"[2] Transition from New to WaitActive"}
,"v":0,{"name":""msgcrucible"":","level":30[1] 1008 final dependency list [JobId(1004)]","v":0,"name":"crucible","level":30,","time"time"::"","2023-09-22T23:14:58.710868803Z2023-09-22T23:14:58.710876736Ztime""":",,"2023-09-22T23:14:58.710882463Z"hostname,"":""hostname":ip-10-150-1-55.us-west-2.compute.internal""hostname":",ip-10-150-1-55.us-west-2.compute.internal""pid,"":pidip-10-150-1-55.us-west-2.compute.internal""4759:,}4759
,""{":"pid""downstairsmsg""}:
":4759{"}msg
"[2] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) Active Active WaitActive ds_transition to WaitQuorum:"","v":0,"name":"[1] 1013 final dependency list [JobId(1008), JobId(1004), JobId(1012)]"{,crucible""v",:"0,level"":name30":""msg"crucible":","level":30[1] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) Active WaitActive New ds_transition to WaitQuorum",",time":""v",2023-09-22T23:14:58.71093409Z""time",":"hostname:"2023-09-22T23:14:58.710939484Z0":",,""name"ip-10-150-1-55.us-west-2.compute.internalhostname"",:""pid"::4759ip-10-150-1-55.us-west-2.compute.internal"},"
pid"":{4759crucible",","msg"level""::""":downstairs"30[2] Transition from WaitActive to WaitQuorum}"
,"v":0,"name":"crucible","level":30,",time":""time":2023-09-22T23:14:58.710988368Z"","hostname"2023-09-22T23:14:58.710982068Z":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid"}:
4759}
{"msg":"{"msg":"[2] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) Active Active WaitQuorum ds_transition to Active","v":[1] Transition from WaitActive to WaitQuorum"0,,""namev"":":crucible"0,","level"name":":crucible30","level":30,"time":"2023-09-22T23:14:58.711046491Z","hostname":",ip-10-150-1-55.us-west-2.compute.internal"","time":"pid":47592023-09-22T23:14:58.711049274Z"}
,"hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759"msg":"}
[2] Transition from WaitQuorum to Active","v":0,"name":"{crucible","level":30"msg":"[1] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.711095385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time{":"2023-09-22T23:14:58.711108184Z""msg",":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":47594dc045ec-5a1e-410e-8896-79fc1c563313 is now active with session: 67ef3947-ce9c-400e-a753-67511f30592a",}"
v":0,"name":"crucible"{,"level":30"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.711147725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.711157633Z",""hostname":msg":""{ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) Active Active Active ds_transition to Faulted}"
msg"":","v{":0,"Crucible stats registered with UUID: 0a679673-3132-4061-840f-f267f0742c60"name":","cruciblev":""0msg",,":"level"":name"30:"crucible"[2] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) Active Active New ds_transition to WaitActive",",level"":v"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.711212279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,{,"""timetime":msg""::"""[1] Transition from Active to Faulted"2023-09-22T23:14:58.711219777Z"2023-09-22T23:14:58.711225807Z",",","hostnamev"hostname"":::0"","name":"test live_repair::repair_test::test_repair_dep_cleanup_repair ... ip-10-150-1-55.us-west-2.compute.internalcrucible"ok,
"level":"30,ip-10-150-1-55.us-west-2.compute.internal""pid",:"4759pid":4759}
}
{,"time":""2023-09-22T23:14:58.711283829Z"{msg",":"hostname"":msg"":"Crucible 0a679673-3132-4061-840f-f267f0742c60 has session id: fde1a8d3-ce20-48f0-916c-50cd4aa6658c"ip-10-150-1-55.us-west-2.compute.internal",[2] Transition from New to WaitActive"",v,""":pid"0:v4759",}":
name"0:","crucible{"name":",""cruciblemsg"":","level"level"::3030[1] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.711347501Z",","time":"hostname":",2023-09-22T23:14:58.711347847Z""ip-10-150-1-55.us-west-2.compute.internal,time""":",hostname"2023-09-22T23:14:58.711355499Z""pid":",":hostname4759":"}ip-10-150-1-55.us-west-2.compute.internal"
,"ip-10-150-1-55.us-west-2.compute.internal"pid","{pid"::47594759"msg":}"
}
{[0] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) New New New ds_transition to WaitActive","v":"0msg"{:","name":"crucible"[1] Transition from Faulted to LiveRepairReady",",""level"v"::0msg",":"name":"30crucible","level":30[2] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time",:""time":"2023-09-22T23:14:58.711432992Z"2023-09-22T23:14:58.711435449Z",","hostname":"hostname":","ip-10-150-1-55.us-west-2.compute.internal"time","ip-10-150-1-55.us-west-2.compute.internal"pid":",:"2023-09-22T23:14:58.711443623Zpid"":4759,4759"}hostname}"
:"
ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759{"}
msg":""msg":"{[0] Transition from New to WaitActive","v":[1] 4dc045ec-5a1e-410e-8896-79fc1c563313 (67ef3947-ce9c-400e-a753-67511f30592a) Active LiveRepairReady Active ds_transition to LiveRepair""0,msg":","v":name":""0crucible",[2] Transition from WaitActive to WaitQuorum"",",name":"level"crucible"",:v30""level"::030,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.711530899Z","time,"":hostname"":",2023-09-22T23:14:58.711535164Z""ip-10-150-1-55.us-west-2.compute.internal,"time"hostname":"":","ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.711539154Z,"pid"pid""::47594759,"}hostname
":"}
ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759{"}msg
"":msg":""{[1] Transition from LiveRepairReady to LiveRepair","v""[0] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) WaitActive New New ds_transition to WaitQuorum":msg":,""v":00,,""name":name":"[2] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) Active Active WaitQuorum ds_transition to Active"crucible",",crucible"","level":level"30:"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.711623661Z",",hostname":""time":ip-10-150-1-55.us-west-2.compute.internal"","pid":47592023-09-22T23:14:58.711622339Z",},""
hostname"time":"{:"2023-09-22T23:14:58.711629161Z""msg":"ip-10-150-1-55.us-west-2.compute.internal",[0] Transition from WaitActive to WaitQuorum"","hostnamev",":pid"":0:"4759,"name"ip-10-150-1-55.us-west-2.compute.internal:"}crucible"
",","level":pid"30{:{4759"}msg
":""msg",:""time"[1] 0 final dependency list []":"{Crucible stats registered with UUID: 7670fa61-da3c-40aa-a375-f47901b91c30"2023-09-22T23:14:58.711682974Z",",,""v"vmsg":""hostname:"0:,"""nameip-10-150-1-55.us-west-2.compute.internal"":",[2] Transition from WaitQuorum to Active""cruciblepid"":,4759",:}level
"0",{:"30"name"v"::msg":"0","crucible"name",:""level"crucible[0] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) WaitQuorum New New ds_transition to Active"":,,""v,"":time0",":name""level"2023-09-22T23:14:58.711728766Z:""30:crucible,"",hostname""30level:"":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{,"2023-09-22T23:14:58.711760623Z""time",msg"":hostname""::"","Crucible 7670fa61-da3c-40aa-a375-f47901b91c30 has session id: d9c136ec-387e-4ef7-9b2c-d54c0a404272ip-10-150-1-55.us-west-2.compute.internal"","time"2023-09-22T23:14:58.711755674Z":v,"":pid0",:"4759name"}:
""crucible"{,2023-09-22T23:14:58.711760012Z",msg""level:"":"30[0] Transition from WaitQuorum to Active","",hostname""hostname"::""v":,0",time""name:"":ip-10-150-1-55.us-west-2.compute.internal"",crucible"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:14:58.711805485Z,"","pid"",hostnamelevel""::"30:"4759ip-10-150-1-55.us-west-2.compute.internal"pid",",:"pid,"":time4759":}"
2023-09-22T23:14:58.711830957Z"4759{,"":}hostname"":"msg":""
ip-10-150-1-55.us-west-2.compute.internal[0] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) New New New ds_transition to WaitActive"",downstairs",""pidv""::4759{}0}
",
"msg{{name""":msg"":crucible""",msg"":level:"[1] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) Active New New ds_transition to WaitActive"":,30""v":08ee74e9c-4380-4bf2-913b-9825b6e2923a is now active with session: 7c4ead5b-3477-4337-95df-a84bbad26641[1] 0 final dependency list [JobId(1006)]"",","namev":,""",:crucibletime"",:""level"":v"302023-09-22T23:14:58.711891071Z":,"00hostname,",":"name"":"name"crucible":,"",ip-10-150-1-55.us-west-2.compute.internal""time,"":"pid":level47592023-09-22T23:14:58.711909345Z"}crucible"":
,30","hostname{level""::""msg":30"ip-10-150-1-55.us-west-2.compute.internal","pid":[0] Transition from New to WaitActive"4759,"}v
":0,"{name":""crucible"msg":,""level":30[1] Transition from New to WaitActive","v":0,",name"":time"":"crucible","2023-09-22T23:14:58.711942859Z,level",:"30time":"""2023-09-22T23:14:58.711958688Z"time",,":hostname,""hostnametime"""::"":""2023-09-22T23:14:58.711971119Z"ip-10-150-1-55.us-west-2.compute.internal",","hostnamepid""::"4759ip-10-150-1-55.us-west-2.compute.internal"}2023-09-22T23:14:58.711949094Zip-10-150-1-55.us-west-2.compute.internal
",,{"""pidmsg""::4759",}""pid"hostname":"[0] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) WaitActive New New ds_transition to WaitQuorum
"ip-10-150-1-55.us-west-2.compute.internal"{,":,"4759pid":"vmsg""::0",4759,""name"}"[1] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) Active WaitActive New ds_transition to WaitQuorum:"",crucible""v,"":
:level0",:"30name":"{crucible","level":30""downstairs"msg":"}
,,""timetime""::""[0] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) Active Active Active ds_transition to Faulted"{2023-09-22T23:14:58.712042969Z2023-09-22T23:14:58.712039818Z,""""v",,"":hostnamehostname""::""msg":"0ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759[1] 0 final dependency list [JobId(1007), JobId(1006)]"}}

{,"{,"""msgmsg""::""v":name0[0] Transition from WaitActive to WaitQuorum[1] Transition from WaitActive to WaitQuorum"",,"""v,"":v0",:"0name":",name""name:"":crucible"crucible"crucible"",,""level"level:"30::,""30level":crucible"30,"level":30,"time":",2023-09-22T23:14:58.712112894Z""time",:""hostname":"2023-09-22T23:14:58.712116124Z","hostnameip-10-150-1-55.us-west-2.compute.internal"":","pid":4759,"}ip-10-150-1-55.us-west-2.compute.internal
"time,":"{"pid2023-09-22T23:14:58.712118652Z""":msg4759",",}:""
hostname":"{time":ip-10-150-1-55.us-west-2.compute.internal[1] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) Active WaitQuorum New ds_transition to Active""msg":,""v"":"02023-09-22T23:14:58.712121534Z,,"[0] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) WaitQuorum New New ds_transition to Active"name,"":v"":"0crucible,,""pid":name"",:""4759hostname"level"":crucible:"30,"","level"":"ip-10-150-1-55.us-west-2.compute.internal"downstairs",:,30""time}"
:,""pid"time2023-09-22T23:14:58.712176902Z"":,"":hostname4759"2023-09-22T23:14:58.712186174Z:"","}hostname"ip-10-150-1-55.us-west-2.compute.internal:"","
pidip-10-150-1-55.us-west-2.compute.internal"":,"4759pid":}4759
}{
{""{msg":""msg"msg[1] Transition from WaitQuorum to Active"":":","[0] Transition from WaitQuorum to Activev"":,0[0] Transition from Active to Faulted",""v,"name"":v"0:,"":namecrucible0"":",",cruciblename":"""crucible,level"":level30":"30,"level":30,",time"":time"":"2023-09-22T23:14:58.712242754Z"2023-09-22T23:14:58.712244432Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal",ip-10-150-1-55.us-west-2.compute.internal""pid,"":pid4759":4759}
},"
{time":{"""msg"msg:"":"2023-09-22T23:14:58.712246859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"[1] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) Active New New ds_transition to WaitActive[2] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) Active Active New ds_transition to WaitActive"",,""vv""::00,,""namename""::""crucible"crucible,"",level"":level30"::304759}
,","time"time:"":"{2023-09-22T23:14:58.712291046Z2023-09-22T23:14:58.712292787Z"",,""hostnamehostname""::"""msg":"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:":47594759}}

{[0] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) Faulted Active Active ds_transition to LiveRepairReady"{",msg"""msgv:"":""[1] Transition from New to WaitActive":,"[2] Transition from New to WaitActivev"",:"0v,"":00name,"":name"":"crucible"crucible,"",,level""level"::3030"name":"crucible","level":30,,""timetime""::""2023-09-22T23:14:58.712338785Z2023-09-22T23:14:58.712339025Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{,"time"{msg":"""msg:"":"2023-09-22T23:14:58.712343592Z","[2] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) Active Active WaitActive ds_transition to WaitQuorumhostname"":"[1] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) Active WaitActive New ds_transition to WaitQuorum,""v,"":v"0:,0","namename""::""crucible"crucible,""level,""level:"30:ip-10-150-1-55.us-west-2.compute.internal"30,"pid":4759}
,,""timetime""::""{2023-09-22T23:14:58.712392492Z2023-09-22T23:14:58.712390748Z"",,""hostname"hostname:"":""msg":"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759[0] Transition from Faulted to LiveRepairReady"}}

{test live_repair::repair_test::test_repair_dep_cleanup_some ... ,"{msg":""msg"ok
[2] Transition from WaitActive to WaitQuorum:""",v"[1] Transition from WaitActive to WaitQuorum""v",:"0v,"":name0",:""name":crucible:"","crucible"level,"":level30"0:30,"name":"crucible","level":30,","time"time:"":"2023-09-22T23:14:58.712447646Z"2023-09-22T23:14:58.712449327Z",","hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid:":47594759}}

{,"{"time"msgmsg""::""":"2023-09-22T23:14:58.712453911Z","hostname":"[1] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) Active WaitQuorum New ds_transition to Active[2] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) Active Active WaitQuorum ds_transition to Active""ip-10-150-1-55.us-west-2.compute.internal",,""vv":"0:,0",,name""name:"":"crucible"crucible"",,pid"""levellevel""::3030:4759}
,,""timetime""::""{2023-09-22T23:14:58.712500134Z2023-09-22T23:14:58.712499968Z"",,""hostnamehostname"""::""msg":"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{[0] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) LiveRepairReady Active Active ds_transition to LiveRepair"{",msg""msg:"":""v":[1] Transition from WaitQuorum to Active"[2] Transition from WaitQuorum to Active",0,""vv""::00,,""name":name"":crucible"",crucible""level,"":level30":,30"name":"crucible","level":30,"time",:""time":"2023-09-22T23:14:58.712550309Z",2023-09-22T23:14:58.712552113Z"",hostname"":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internal"pid",:"4759pid":}4759
}
{,{"""msgmsg""::""time":"2023-09-22T23:14:58.71255665Z","hostname":"0a679673-3132-4061-840f-f267f0742c60 is now active with session: 30d6992e-fe0e-4614-b657-ff6d5d0c30d2"[2] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) Active Active New ds_transition to WaitActive",","ip-10-150-1-55.us-west-2.compute.internal"vv""::00,,""name"name:"":"crucible"crucible,"",,level""level:"30:"30pid":4759}
,,""timetime""::""{2023-09-22T23:14:58.712599825Z2023-09-22T23:14:58.712598427Z"",,""hostname"hostname:""":"msg":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:[0] Transition from LiveRepairReady to LiveRepair"4759":,}4759
}"
{v":{"0msg""msg:"":,""name":"crucible"[2] Transition from New to WaitActive",",level[0] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) Active Active Active ds_transition to Faulted""v",":v0",:"0",name"":name:"":"crucible"crucible,"","level"level:"30:3030,,""timetime""::""2023-09-22T23:14:58.712650299Z2023-09-22T23:14:58.712651435Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:"4759:,}4759
}
{"time""msg{":"":"msg":"2023-09-22T23:14:58.71265145Z","[0] Transition from Active to Faultedhostname":"","[2] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) Active Active WaitActive ds_transition to WaitQuorumv"":,0"ip-10-150-1-55.us-west-2.compute.internal"v,"":,"name0",:""name"crucible:"",pid"crucible"":,level"":level"30:475930}
,,""timetime""::""2023-09-22T23:14:58.712700845Z"2023-09-22T23:14:58.712702165Z",,""{hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759"}}

{msg":""{msg":""Waiting for Close + ReOpen jobs"msg":","v":0,"[2] Transition from WaitActive to WaitQuorum"name":"[0] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) Faulted Active Active ds_transition to LiveRepairReady,"",v""v:"0:,0","namename""::""crucible"cruciblecrucible"",,""levellevel""::3030,"level":30,,""timetime""::""2023-09-22T23:14:58.712757323Z2023-09-22T23:14:58.712757166Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid:":47594759},}

{"time"{msg""":msg""::""2023-09-22T23:14:58.712759762Z"[0] Transition from Faulted to LiveRepairReady",,[2] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) Active Active WaitQuorum ds_transition to Active""v",:"0v,":"0",name"hostname""name:""::crucible"""crucible,"","level"level:"30:ip-10-150-1-55.us-west-2.compute.internal"30,"{pid":4759}
,,""time"time:""{":2023-09-22T23:14:58.712809087Z""msg",2023-09-22T23:14:58.712807284Z""",hostname":"hostname:""msg":"":"Crucible stats registered with UUID: 8004c607-ee2d-4e41-b83c-b23601ad76a4"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759,RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]"}},"

{"v""{msg":"":msg[2] Transition from WaitQuorum to Active"":,""0vv":,0",[0] 0a679673-3132-4061-840f-f267f0742c60 (30d6992e-fe0e-4614-b657-ff6d5d0c30d2) LiveRepairReady Active Active ds_transition to LiveRepair""name,"":v""name"crucible:"0,,"":""namelevel""::"30crucible":crucible",0",",level"level"":,:"30time"30:name"":"crucible"2023-09-22T23:14:58.712878543Z",","level",hostname""time:"":":ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.712890126Z"",",pid"":30hostname4759":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
",msg{":"""msg":time"":"7670fa61-da3c-40aa-a375-f47901b91c30 is now active with session: 6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a"2023-09-22T23:14:58.712892122Z,[0] Transition from LiveRepairReady to LiveRepair""v,"":v0,,"":name0",:""name"crucible:"",""levelcrucible"":,30","level"time":":"hostname":"2023-09-22T23:14:58.71290749Z",30"time,ip-10-150-1-55.us-west-2.compute.internal"",":hostname"":""pid"2023-09-22T23:14:58.712937731Z,"","timeip-10-150-1-55.us-west-2.compute.internalhostname""::"":"2023-09-22T23:14:58.712947122Z"ip-10-150-1-55.us-west-2.compute.internal,"",,hostname"""pid:pid"4759""::4759}ip-10-150-1-55.us-west-2.compute.internal}"
,"
4759{pid":4759}}"
msg":{"{
"[0] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) Active Active Active ds_transition to Faultedmsg""",:"msg"{:v""":Waiting for Close + ReOpen jobs0",",name""v":Crucible 8004c607-ee2d-4e41-b83c-b23601ad76a4 has session id: aebb10c0-c584-4415-ac78-481406e247a9""":crucible0",,""namelevel""msg::,"""30vcrucible"":":,"0level",":,30"RE:0 close id:1000 queued, notify DS"time"name:",""2023-09-22T23:14:58.713021333Z":"v"crucible",,""time"hostname:"":",":ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:14:58.713030212Z"",",level0""pidhostname""::"4759:,}
ip-10-150-1-55.us-west-2.compute.internal"30"{,name":"""crucible"pidmsg""::"4759,"}level"
[0] Transition from Active to Faulted":,{"30"v"msg:":0","name":"crucible",RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]""level,"":v30":,"0,time"":name"":"crucible"2023-09-22T23:14:58.713062633Z",,"",leveltime""::30"",hostname"2023-09-22T23:14:58.713080863Z:"",""time":ip-10-150-1-55.us-west-2.compute.internal"","hostname2023-09-22T23:14:58.713075992Z",:"pid"time"":""ip-10-150-1-55.us-west-2.compute.internal":2023-09-22T23:14:58.713093688Z,"",pid"",hostname:"4759:"}"
ip-10-150-1-55.us-west-2.compute.internal"4759,{"hostname":"}"pidmsg"
"::"4759ip-10-150-1-55.us-west-2.compute.internal"}
{,"pid{[0] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) Faulted Active Active ds_transition to LiveRepairReady"",:"""vmsg"":msg"4759"::0",RE:0 close id:1000 queued, notify DS""name",:""v"crucible:}0",,""
[0] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) New New New ds_transition to WaitActive"levelname""::30,""{crucible"v",":level0"":,30",time":""msg"2023-09-22T23:14:58.713176649Zname"",:",""crucible"hostnametime",":"":":"level":2023-09-22T23:14:58.713186436Zip-10-150-1-55.us-west-2.compute.internal30RE:0 Wait for result from close command 1000:1""",,""pid"hostname:"4759:",}"
ip-10-150-1-55.us-west-2.compute.internal"v":{,0"",pidmsg""":,:4759"name"}[0] Transition from Faulted to LiveRepairReady"
,""{time":""vmsg""::0",2023-09-22T23:14:58.713206207Z"RE:0 Wait for result from close command 1000:1name""",":,"":cruciblehostname":"v,"":level0",:"30""name"crucible":ip-10-150-1-55.us-west-2.compute.internal",,""pid":"4759level"}:
,crucible""30,time""level:""{:302023-09-22T23:14:58.713252789Z",""msghostname"":":"[0] Transition from New to WaitActive","v":0ip-10-150-1-55.us-west-2.compute.internal",",name":",""pidtime""::4759"crucible"}2023-09-22T23:14:58.713273236Z
",,,{""""hostnamemsg""::""level":time"30:ip-10-150-1-55.us-west-2.compute.internal"",[0] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) LiveRepairReady Active Active ds_transition to LiveRepair""pid",:"4759v"}:
0,"2023-09-22T23:14:58.713270314Ztime":,""name":"2023-09-22T23:14:58.713303832Z"crucible"",",,level":"30hostname":""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}ip-10-150-1-55.us-west-2.compute.internal"
,"pid{,""time":"msg":"2023-09-22T23:14:58.713325598Z"":,"4759hostname":"[0] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) WaitActive New New ds_transition to WaitQuorum"},"
ip-10-150-1-55.us-west-2.compute.internal"v":,"0pid":,"4759name":"}crucible"
,"level":30{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":","crucible"time":","level":2023-09-22T23:14:58.713363012Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":,""time":"[0] Transition from WaitActive to WaitQuorum"2023-09-22T23:14:58.713374295Z",","v"hostname":":0,"name":"ip-10-150-1-55.us-west-2.compute.internalcrucible"",,""pid":level":304759}
{"msg":"Waiting for Close + ReOpen jobs",,""time":"v":0,"2023-09-22T23:14:58.713397479Z"name,"":"hostname"crucible":","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":"2023-09-22T23:14:58.713414198Z","hostname":"[0] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) WaitQuorum New New ds_transition to Active","v"ip-10-150-1-55.us-west-2.compute.internal":,0","pid":name":"4759crucible"},"
level":30{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:14:58.713435539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:14:58.713443103Z""msg":","hostname":"[0] Transition from WaitQuorum to Active","v"ip-10-150-1-55.us-west-2.compute.internal":,"0pid":,"4759name":"}crucible"
,"level":30{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713467356Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759},"
time":"{2023-09-22T23:14:58.713472743Z"",msg":""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) Active New New ds_transition to WaitActive"}
,"v":0{,"name"":"msg"crucible:"","level":30RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713498343Z","hostname":","time":ip-10-150-1-55.us-west-2.compute.internal"","pid":47592023-09-22T23:14:58.713501613Z"}
,"hostname":"{ip-10-150-1-55.us-west-2.compute.internal",""pidmsg":"":4759}
[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.71353096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713553896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713572254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713592498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713610511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.71363059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713648725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713668999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713687331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713707426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713725631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"8004c607-ee2d-4e41-b83c-b23601ad76a4 is now active with session: a534a5bb-c489-4ad8-9603-6af8970f56bd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713745028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713765029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713782878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713803335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.71382153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.71384199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713864372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713883351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713901674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713919672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:58.713937831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.578602998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.578691602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.580733063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.580792493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.58122197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.581277113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 04482c5c-f564-49f3-897c-51c4de58239a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.582548977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:36921 in state Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.582603806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.582646357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 04482c5c-f564-49f3-897c-51c4de58239a, session_id: cc12d06e-bd83-45fe-987f-779d8c6339ec, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.582784891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] upstairs guest_io_ready=TRUE, promote! session cc12d06e-bd83-45fe-987f-779d8c6339ec","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.58284283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: 04482c5c-f564-49f3-897c-51c4de58239a, session_id: cc12d06e-bd83-45fe-987f-779d8c6339ec, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.582917376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583079207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] downstairs client at 127.0.0.1:36921 has UUID d057f0d7-1fef-421b-b789-cb2746bfdb26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583134292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: d057f0d7-1fef-421b-b789-cb2746bfdb26, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583196597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Returning client:0 UUID:d057f0d7-1fef-421b-b789-cb2746bfdb26 matches","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583243694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
waiting for ds1 message in test_replay_occurs
{"msg":"[0] send last flush ID to this DS: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583288107Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"04482c5c-f564-49f3-897c-51c4de58239a Offline Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583332338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet LastFlush { last_flush_number: JobId(0) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.58337714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] Replied this last flush ID: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583420616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 04482c5c-f564-49f3-897c-51c4de58239a (cc12d06e-bd83-45fe-987f-779d8c6339ec) Offline Active Active ds_transition to Replay","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583467824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from Offline to Replay","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583510665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 04482c5c-f564-49f3-897c-51c4de58239a Transition from Replay to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583554933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 04482c5c-f564-49f3-897c-51c4de58239a (cc12d06e-bd83-45fe-987f-779d8c6339ec) Replay Active Active ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583601617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from Replay to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583645978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583687625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 127.0.0.1:36921 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583730403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"04482c5c-f564-49f3-897c-51c4de58239a Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583770841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.583812525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.58851288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.588624351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:59.711132074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:59.711240044Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.711289225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.711333825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 8ee74e9c-4380-4bf2-913b-9825b6e2923a (7c4ead5b-3477-4337-95df-a84bbad26641) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.711381359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.711423925Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.711464258Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.711504008Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:14:59.711544174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.711583177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.712830027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.712881207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Repair for extent 0 s:1 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.712922645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.712965115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.713393291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.713442059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.713492194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.714478959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.714529743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:14:59.714569986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] flow control start ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:59.817088951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] flow control start ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:59.81714109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] flow control start ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:14:59.817161372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
waiting for ds1 message in test_replay_occurs
{"msg":"[0] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:00.585347674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.585436232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.585479848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.712768486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:00.712826236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.712847274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the repair job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.713292556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.713347144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.713444219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.713778062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.713844837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.714951795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:00.715003906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:00.817811058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[2] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:00.8179158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[1] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:00.819029685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
test dummy_downstairs_tests::protocol_test::test_replay_occurs ... ok
{"msg":"Crucible stats registered with UUID: 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587204571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 has session id: 05b118b3-d9d3-49db-ae4e-8cebbfe05ada","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587260194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587283175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587302665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587323736Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587342551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587362903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587381068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587401157Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587419289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587439877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.58745832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.58747876Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587496952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587517361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587535323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587563945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587583878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.58760424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587622296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 is now active with session: 34c41cf8-0433-4cf1-9c64-d266d05b21c9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587641814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587662158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587680014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587700246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587718451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3d04bbe2-2f14-4f9e-87aa-ab9edd65bb25 (34c41cf8-0433-4cf1-9c64-d266d05b21c9) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587738857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587757075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587774936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587793446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587811322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.587829341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.714035237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.714091409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.714111872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.714130487Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:01.714157316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.714177847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:01.714196516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:01.71421465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0   Err  Done  Done  false
    2    Acked  1001    NoOp       0  Skip  Done  Done  false
    3    Acked  1002    NoOp       0  Skip  Done  Done  false
    4    Acked  1003  Reopen       0  Skip  Done  Done  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          0      4      4      8
    Skipped       3      0      0      3
    Error         1      0      0      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.714766213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.714822222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.714865588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the repair job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.714903545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.714944768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: ab17680f-7c63-4c2f-bd24-9eb69ab45861","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.715046973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ab17680f-7c63-4c2f-bd24-9eb69ab45861 has session id: 6a03f4da-4cdc-4031-8c63-9ab921ddeafd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.71509934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",{"pid":4759}
"msg":"{Now move the NoOp job forward","v""msg"::"0,"name":"crucible","level":[0] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) New New New ds_transition to WaitActive30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.715147304Z","hostname":,""time":"ip-10-150-1-55.us-west-2.compute.internal","pid"2023-09-22T23:15:01.715141991Z":4759,"}hostname"
:"{ip-10-150-1-55.us-west-2.compute.internal","pid":"4759msg":"}
[0] Transition from New to WaitActive","v":0,"name":"{crucible","level":30"msg":"[0] DS Reports error Err(GenericError(\"bad\","time":"2023-09-22T23:15:01.715198192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759)) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"},
{"msg":""[0] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) WaitActive New New ds_transition to WaitQuorum"v",:"0v":,0","name"name:"":"crucible"crucible",","level"level"::5030,"time":","time2023-09-22T23:15:01.715269558Z"":","hostname":"2023-09-22T23:15:01.715269537Z","ip-10-150-1-55.us-west-2.compute.internal"hostname",":"pid":4759ip-10-150-1-55.us-west-2.compute.internal"},"
pid":4759,"":"{downstairs"}
"msg":"[0] Transition from WaitActive to WaitQuorum","v":{0,"name":"crucible",""level":msg":30"[0] Reports error GenericError(\"bad\","time":") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"2023-09-22T23:15:01.715331264Z",",v"":hostname":"0,"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible,""pid":,4759"level"}:
50{"msg":"[0] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,","time"time":":"2023-09-22T23:15:01.715377235Z","hostname":"2023-09-22T23:15:01.715367038Z",ip-10-150-1-55.us-west-2.compute.internal",""pid":4759hostname"}
:"{"msg":"ip-10-150-1-55.us-west-2.compute.internal","[0] Transition from WaitQuorum to Active"pid",":v4759":0,"name,"":""crucible":,""level":30downstairs"}
,"time":"2023-09-22T23:15:01.715411867Z",{"hostname":"ip-10-150-1-55.us-west-2.compute.internal""msg,"":pid":"4759{}
[0] client skip 4 in process jobs because fault","{"msg"v"":"msg":":0Crucible stats registered with UUID: dc10ceaf-6b5e-482c-b452-5b861e163a08","name",:"[1] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) Active New New ds_transition to WaitActive""v":,"cruciblev"",0:,0",""namenamelevel""":":crucible:""30,"crucible"level":,30"level":30,"time":"2023-09-22T23:15:01.715466099Z",",hostname"":"time":"ip-10-150-1-55.us-west-2.compute.internal","pid":47592023-09-22T23:15:01.715463483Z}"
,"{hostname":"",msg""ip-10-150-1-55.us-west-2.compute.internal":,time":"""pid"2023-09-22T23:15:01.715468097Z:[1] Transition from New to WaitActive"4759",",,"""hostname":":v":"ip-10-150-1-55.us-west-2.compute.internal"downstairs0,",""pid"name"::}4759
}"
crucible"{,"level":30{"msg":""msg":"[0] changed 1 jobs to fault skipped","v":Crucible dc10ceaf-6b5e-482c-b452-5b861e163a08 has session id: cf4f1db3-0dbf-449f-a8a4-2771f7de8587",0,","v":"timename0",":"name"":":"cruciblecrucible"",,"2023-09-22T23:15:01.715523004Z"level"",level""::hostname3030":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":"2023-09-22T23:15:01.715553249Z","hostname":"[1] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) Active WaitActive New ds_transition to WaitQuorum",ip-10-150-1-55.us-west-2.compute.internal",""time,""pid":v":4759:0},"
"name":"crucible","level":2023-09-22T23:15:01.715553994Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,",time":""":"2023-09-22T23:15:01.715582233Z","downstairshostname"":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg"":"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30[0] 7670fa61-da3c-40aa-a375-f47901b91c30 (6b3a8f2e-9943-4b3e-b0b5-d3cbb5e6085a) LiveRepair Active Active ds_transition to Faulted","v":0,"name":","crucible"time,":""level2023-09-22T23:15:01.715609802Z"",":hostname":30"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) Active WaitQuorum New ds_transition to Active","v":0,",name":""crucible","time":level":30"2023-09-22T23:15:01.715624882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal,""time,":""pid"2023-09-22T23:15:01.715641346Z":,"4759hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{{"msg"":""msg":msg""[0] Transition from LiveRepair to Faulted":","v":[1] Transition from WaitQuorum to Active"[0] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) New New New ds_transition to WaitActive,"","vv":"0:00,,",""namename":"namecrucible"","":"levelcrucible"::"30,""levelcrucible"",":30level":30,","time":"time":"2023-09-22T23:15:01.715699718Z"2023-09-22T23:15:01.715703768Z,"","timehostname",":":""hostname":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:15:01.715704791Zpid":ip-10-150-1-55.us-west-2.compute.internal4759"},""
pid":{4759,"}"
hostname"msg"::""{"msg":ip-10-150-1-55.us-west-2.compute.internal"[2] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) Active Active New ds_transition to WaitActive"",","[0] Transition from New to WaitActive"v"pid:,"0",":v":0name"4759:",}"crucible"
,"namelevel""::{"30crucible""msg,""level":30:","time":"Extent 0 close id:1002 Failed: Error: bad"2023-09-22T23:15:01.715774872Z",,""hostname":"v":ip-10-150-1-55.us-west-2.compute.internal"0,",",time":"pid":2023-09-22T23:15:01.715779088Z"4759,""}hostname
name"":":{"ip-10-150-1-55.us-west-2.compute.internal"crucible"",msg",""pid"::"level"4759[2] Transition from New to WaitActive"},
:"50v":{0,"name""msg":":"crucible","level":30[0] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level",:30,""time":"time":"2023-09-22T23:15:01.715827563Z","hostname":"2023-09-22T23:15:01.71582163Z","ip-10-150-1-55.us-west-2.compute.internal"hostname":,"","pidtimeip-10-150-1-55.us-west-2.compute.internal":"":"2023-09-22T23:15:01.715833811Z,4759"}","
pid":{4759hostname":}""
msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759{}
"[2] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) Active Active WaitActive ds_transition to WaitQuorummsg"":","{v":0RE:0 Wait for result from reopen command 1003:4",""name",:""cruciblemsg":"v""[0] Transition from WaitActive to WaitQuorum,"":level0",:30","name"v"::0","name":"crucible"crucible",,""time,""level":level"30:":302023-09-22T23:15:01.715891534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:01.715908519Z"",msg":""hostname":"[2] Transition from WaitActive to WaitQuorum"ip-10-150-1-55.us-west-2.compute.internal",",v""pid"::04759,",}"nametime"":":crucible""
,"2023-09-22T23:15:01.71591104Z{"level"",:msg"":"hostname"30:"[0] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) WaitQuorum New New ds_transition to Active"ip-10-150-1-55.us-west-2.compute.internal",",v":0",",pidtime":"""name":2023-09-22T23:15:01.71594948Z"4759:","}crucible"hostname,
"":"level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":,""time":"2023-09-22T23:15:01.715975135Z","hostname":"[2] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) Active Active WaitQuorum ds_transition to Active"ip-10-150-1-55.us-west-2.compute.internal",","v":0,"pid":name":"4759crucible","}
level":30{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.715998161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:01.716004311Z""msg,"":hostname":""ip-10-150-1-55.us-west-2.compute.internal"[2] Transition from WaitQuorum to Active,""pid":,"4759v":0},"
name":"crucible"{,"level":"30msg":"[1] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) Active New New ds_transition to WaitActive","v":0,"name":"crucible",","leveltime":"":302023-09-22T23:15:01.716031775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time{":""msg2023-09-22T23:15:01.716041155Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pidab17680f-7c63-4c2f-bd24-9eb69ab45861 is now active with session: 76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb"",":v":47590,"}name"
:"crucible",{"level":"30msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.716066316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,"4759time":"}
2023-09-22T23:15:01.716070815Z","hostname{":""msgip-10-150-1-55.us-west-2.compute.internal","":"pid":4759}
[0] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) Active Active Active ds_transition to Faulted","v{":0,""name"msg":":"crucible","level":30[1] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.716101625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid",:4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.716122618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"time":"2023-09-22T23:15:01.716106531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":"0,msg"":"name":"crucible","level":30[0] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.716187678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:01.716194511Z""msg":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) Active WaitQuorum New ds_transition to Active"},"
v":0,"name":"crucible",{"level":30"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.716220641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{"2023-09-22T23:15:01.716225701Zmsg":"","hostname":"[1] Transition from WaitQuorum to Active","v":0ip-10-150-1-55.us-west-2.compute.internal",","name":"pid"crucible":,"4759level":30}
{"msg":","time":"2023-09-22T23:15:01.7162488Z"[0] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) LiveRepairReady Active Active ds_transition to LiveRepair",","hostname":"v":0,"name":ip-10-150-1-55.us-west-2.compute.internal"",crucible"","pid":level":475930}
{"msg":","time":"[2] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) Active Active New ds_transition to WaitActive"2023-09-22T23:15:01.716269728Z",","v":hostname0,"":"name":"crucible","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:01.716287178Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[0] Transition from LiveRepairReady to LiveRepair"{,"{"v"msg":":0","msgname""::""[2] Transition from New to WaitActive",crucibleNow move the NoOp job forward"","",level""v:"30:v":0,"0name":,""crucible"name":","cruciblelevel":"30,"level",:"30time":"2023-09-22T23:15:01.716316813Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:15:01.716326575Z"pid":,"4759hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}{,
""msgtime""::""{"Waiting for Close + ReOpen jobs2023-09-22T23:15:01.716329012Z"","msg":",v"":0hostname",:""name":"crucible"[2] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) Active Active WaitActive ds_transition to WaitQuorum"ip-10-150-1-55.us-west-2.compute.internal,"",,""levelpid""::304759v":0,"}name
":"crucible","level":30,"time":"2023-09-22T23:15:01.716366173Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:15:01.716374616Zpid":"4759,"hostname}":"
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}{
"msg"{:{""msg":"[2] Transition from WaitActive to WaitQuorum"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]""msg",:""v",":[0] DS Reports error Err(GenericError(0\"v":,bad"\"name"0:,""name":"cruciblecrucible"",",level"":30level":30,"time":")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"2023-09-22T23:15:01.716416255Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal",v""time:"0:,",""pid":2023-09-22T23:15:01.716417375Zname"",:""4759hostnamecrucible"}
":","{level":ip-10-150-1-55.us-west-2.compute.internal"50,""pidmsg":"":4759}
[2] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) Active Active WaitQuorum ds_transition to Active","v":0,"{,name":"""msg"time:"":"crucible","level"RE:0 close id:1000 queued, notify DS2023-09-22T23:15:01.716449167Z"":,,""30vhostname""::"0,"name":"ip-10-150-1-55.us-west-2.compute.internalcrucible"",","pid":level"4759:30,"":","downstairs"time":"}
2023-09-22T23:15:01.716469711Z","hostname":"{,ip-10-150-1-55.us-west-2.compute.internal"","time""msg:"":"pid":47592023-09-22T23:15:01.716475181Z"[0] Reports error GenericError(,\""bad\"hostname"}
:"{"msg":"ip-10-150-1-55.us-west-2.compute.internal","[2] Transition from WaitQuorum to Active"pid,"":v4759":0},"
) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"name":",{crucible"""vmsg""::0",,""level":nameRE:0 Wait for result from close command 1000:1":"",30crucible""v",:"0level,"":50name":"crucible","level":30,"time":"2023-09-22T23:15:01.716521778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,,""{timetime""::"""msg":"2023-09-22T23:15:01.716527259Z2023-09-22T23:15:01.716524572Z"",,""hostnamehostname""::""dc10ceaf-6b5e-482c-b452-5b861e163a08 is now active with session: 1ac6e38b-5a66-4230-80ea-1ef440760337","v":0,"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759name":},
"""crucible":","level":downstairs"30}
{"msg":","time":"2023-09-22T23:15:01.7165607Z"[0] client skip 4 in process jobs because fault",","hostname":"v":0ip-10-150-1-55.us-west-2.compute.internal",,""pid":name":4759"}
crucible","{level":"30msg":"[1] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":","time":2023-09-22T23:15:01.716585008Z"","2023-09-22T23:15:01.7165915Z"hostname,""hostname":":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internal"pid":,"4759pid":}4759
,"":"{downstairs"}
"msg":"{[1] Transition from Active to Faulted","v":"0msg":","name":"[0] changed 0 jobs to fault skipped"crucible",",v"":level":300,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.71662991Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid":2023-09-22T23:15:01.716632397Z"4759,"}hostname
":"{ip-10-150-1-55.us-west-2.compute.internal""msg":","pid":4759,"":"downstairs"}
[1] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) Active Faulted Active ds_transition to LiveRepairReady","v{":0,"name":""cruciblemsg":"","level":30[0] 8004c607-ee2d-4e41-b83c-b23601ad76a4 (a534a5bb-c489-4ad8-9603-6af8970f56bd) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.71666223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{"2023-09-22T23:15:01.716667638Z"msg":","hostname":"[1] Transition from Faulted to LiveRepairReady","vip-10-150-1-55.us-west-2.compute.internal"",":pid":04759,"}name":"
crucible","level{":30"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.716693997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:01.716699083Z"",msg":""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] dc10ceaf-6b5e-482c-b452-5b861e163a08 (1ac6e38b-5a66-4230-80ea-1ef440760337) Active LiveRepairReady Active ds_transition to LiveRepair","v":0{,"name":""crucible"msg",":"level":30RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.716730138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:01.716734306Z""msg":","hostname":"[1] Transition from LiveRepairReady to LiveRepair","ip-10-150-1-55.us-west-2.compute.internal","v":pid":04759,"name"}:"
crucible","level":{30"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:01.716761737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:01.716767108Z"","msg"hostname":":"Waiting for Close + ReOpen jobs"ip-10-150-1-55.us-west-2.compute.internal,""v",":pid0",":name":"4759crucible","}level":30
{"msg":"RE:0 Bailing with error","v":0,,""time":name"":"crucible"2023-09-22T23:15:01.716794196Z",,""level":40hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,""timemsg":"":"2023-09-22T23:15:01.716808713Z"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]",,""v":0hostname":","name":"crucible"ip-10-150-1-55.us-west-2.compute.internal",,""pid":level"4759:30}
,"time":"2023-09-22T23:15:01.716831263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.716850497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.716868984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: c96f066c-48e7-437e-b4d0-bca7db257ca7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717222523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible c96f066c-48e7-437e-b4d0-bca7db257ca7 has session id: 2d89a9cc-9693-4b98-b14a-70cd38065182","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717254218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717276491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717295749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717316484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717334859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717355115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717373276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717393501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717411537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717437973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717457662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717478156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717496368Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717516645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.71753468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717555268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717573444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717593821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.71761187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"c96f066c-48e7-437e-b4d0-bca7db257ca7 is now active with session: 777a2755-8455-4e73-a9be-625addff6b55","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717631524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717651723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717669767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717690099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717708461Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717728928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.71774729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.71776962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717789257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717807511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.717825865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] flow control start ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:01.787708968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
test dummy_downstairs_tests::protocol_test::test_flow_control ... ok
{"msg":"Crucible stats registered with UUID: 208c6b08-32db-4498-a111-6f48aa07c8ef","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.843762644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 208c6b08-32db-4498-a111-6f48aa07c8ef has session id: df19bc5a-b192-411a-a6e4-6b07040e819d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.843811723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.843833885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.843852889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.843873602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.843891934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.843912088Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.843930305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.84395073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.843969358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.843990079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844008594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844029049Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.84405553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844077367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844095503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844131336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844152644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844173033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844191049Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"208c6b08-32db-4498-a111-6f48aa07c8ef is now active with session: a9d99eb4-fe49-4eea-a3e5-bed72149e733","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.84421064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844230986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844248978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844269465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844287787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.84430834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844326781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844344702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844363232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844381285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:01.844399539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] flow control start ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:01.857341071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.588866474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.588921115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.588942683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:02.716727741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:02.716798529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.716820824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.716841079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.716863463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.716882512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.716900525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.71691863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:02.716937296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.716955713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.716974856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] ab17680f-7c63-4c2f-bd24-9eb69ab45861 (76bbd7fe-ffcc-4ebf-8060-e1708e4d5ecb) LiveRepair Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.716996548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.717026624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.717046449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.717355116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.717390019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Repair for extent 0 s:2 d:[ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.717409988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.71743057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.717934086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.717966779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.717988145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.845406116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.845460876Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.845480536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:02.845501092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.590307091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.590362235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.715690434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.715775443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:03.715809434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 67c26488-dbae-450b-b463-1bdda5022bc7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716342736Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 67c26488-dbae-450b-b463-1bdda5022bc7 has session id: 4e82f1f9-0353-4e67-9d25-ed3e57002eeb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716379405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.71641268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716433404Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716454758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716473176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716493416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716511637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716531975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716549849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.71657035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716588811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716608909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716626891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.71664711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716665025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716685307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716703608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716724064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716742056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"67c26488-dbae-450b-b463-1bdda5022bc7 is now active with session: 70c18185-953d-4518-93af-6f5485ebb3e0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716766257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716787684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716805826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716826346Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716844527Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716864912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716883343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716901236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716919995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.71693812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.716956467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.717254427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:03.717290545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.717310887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the repair job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.718802203Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.718834693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.71887353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.719369674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.719405329Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:03.846802954Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:03.846867426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.846888933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.846908349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 208c6b08-32db-4498-a111-6f48aa07c8ef (a9d99eb4-fe49-4eea-a3e5-bed72149e733) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.84693032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.846949002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.846967049Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:03.846985333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:03.847003998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.717931867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.717986724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.718008859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.718561694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.71859767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.718618343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.718636654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:04.718655256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.718684903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:04.71870478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:04.718722988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done   Err  Done  false
    2    Acked  1001    NoOp       0  Skip  Skip  Done  false
    3    Acked  1002    NoOp       0  Skip  Skip  Done  false
    4    Acked  1003  Reopen       0  Skip  Skip  Done  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          1      0      4      5
    Skipped       3      3      0      6
    Error         0      1      0      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.719154887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.71920196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.719234454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the repair job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.719263274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.719294038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 740b9b19-2d4a-44eb-b854-682cc860698c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.7194473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 740b9b19-2d4a-44eb-b854-682cc860698c has session id: 285f064a-eb2a-49dd-bff1-97e3d20cec44","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.719504269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.719550939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.719592452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30{,"time":""msg"2023-09-22T23:15:04.719635315Z":","hostname":"Crucible stats registered with UUID: 6b6841a1-369e-4ce0-bd40-27f441691315"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"v":}
0,"name":"crucible{","level"":msg":"30[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30{"msg":"Now move the NoOp job forward",","time"v",:":"2023-09-22T23:15:04.719681536Z"time",:"0",hostname""name:":"2023-09-22T23:15:04.719674741Z"crucible"",","ip-10-150-1-55.us-west-2.compute.internalhostname":"level"":ip-10-150-1-55.us-west-2.compute.internal"30,","pidpid""::47594759}
}
{",msg{"":"time":"2023-09-22T23:15:04.719722939Z","hostname":"[0] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) WaitQuorum New New ds_transition to Active","ip-10-150-1-55.us-west-2.compute.internalv"":,""pid"0msg,""name":::""crucible"4759,"level":}30
,"time":"2023-09-22T23:15:04.719767941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30{"msg":"Crucible 6b6841a1-369e-4ce0-bd40-27f441691315 has session id: 6afb4f20-ee36-42e0-b477-7bcca39adbb5"[1] DS Reports error Err(GenericError(\"bad\",,""v"time:":0","name"2023-09-22T23:15:04.719813863Z":","crucible"hostname":","level":ip-10-150-1-55.us-west-2.compute.internal"30,")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"pid":4759}
,"v":0,"name"{:"crucible","level":"50msg":","time"[1] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) Active New New ds_transition to WaitActive":","v":2023-09-22T23:15:04.719842039Z"0,","name":"hostname":"crucible","level"ip-10-150-1-55.us-west-2.compute.internal":,"30pid",":time"4759:"}2023-09-22T23:15:04.719857884Z
","hostname":"{ip-10-150-1-55.us-west-2.compute.internal",,""time"pid""::"msg"47592023-09-22T23:15:04.719875817Z":",,""":hostname"":[0] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) New New New ds_transition to WaitActive"downstairs""},ip-10-150-1-55.us-west-2.compute.internal""
,"v"pid"::47590{,"}name
""msg:"":"{crucible",[1] Reports error GenericError("\"levelbad"\""msg":":30[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level",":time",:50""time":2023-09-22T23:15:04.71994208Z"","hostname":2023-09-22T23:15:04.719948378Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid":}4759
},
"{time":"2023-09-22T23:15:04.719965842Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal"","[0] Transition from New to WaitActivepid":"4759msg,,"":"":""vdownstairs"":}
0[1] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) Active WaitActive New ds_transition to WaitQuorum,""{name",:"""v"msg":":crucible"[1] client skip 4 in process jobs because fault",0,","v":level0"","name"name":":":crucible"crucible,30""level":,30"level":30,"time":"2023-09-22T23:15:04.720044379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,","":time"":"downstairs","}
2023-09-22T23:15:04.72004235Z"time",:{"""msg"hostname"2023-09-22T23:15:04.720046555Z":":",[1] changed 0 jobs to fault skipped"","hostname"v"::"0ip-10-150-1-55.us-west-2.compute.internal",",ip-10-150-1-55.us-west-2.compute.internal"name":""pidcrucible",","pid"level"::"304759:4759}
},"
time":"{2023-09-22T23:15:04.720096849Z","hostname":"{"ip-10-150-1-55.us-west-2.compute.internal"msg",":pid":"4759","":"msg":"downstairs"[1] Transition from WaitActive to WaitQuorum"}
,"v"[0] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) WaitActive New New ds_transition to WaitQuorum{:"0",msg","":"name"v":":[1] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) LiveRepair Active Active ds_transition to Faultedcrucible0"",,,""name"v":0:",""name":"level":crucible"crucible",30,""level":30level":30,"time":"2023-09-22T23:15:04.720160661Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal":,""pid":4759}
2023-09-22T23:15:04.720158251Z",,""{hostname"time"":":msg":""ip-10-150-1-55.us-west-2.compute.internal"[1] Transition from Active to Faulted",2023-09-22T23:15:04.720162645Z""pid":,4759,"}"
v":hostname0{,""name"":"msg":":"crucible","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) Active WaitQuorum New ds_transition to Active"},
","v"time":":0,"2023-09-22T23:15:04.720214509Z"name",:""hostname":"crucible"{ip-10-150-1-55.us-west-2.compute.internal",,""pid"":msg"level":475930:"}
[0] Transition from WaitActive to WaitQuorum"{,"v"":msg":"0,"nameRE:0 Wait for result from reopen command 1003:4"",",:"v":crucible"0,,"""nametime""::level":""30crucible"2023-09-22T23:15:04.720244054Z",",level":"30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:04.720273848Z","hostname":""msg":","ip-10-150-1-55.us-west-2.compute.internal","[1] Transition from WaitQuorum to Activepid":time"4759,""}:"v"
2023-09-22T23:15:04.720270146Z"{:0,""msg"hostname":",:"Extent 0 close id:1003 Failed: Error: bad""name",":ip-10-150-1-55.us-west-2.compute.internalv":0"","crucible",",name":""pidlevel":crucible30"":,"4759level":50}
,"time":","time":2023-09-22T23:15:04.720329311Z"","{hostname":"2023-09-22T23:15:04.720326105Z","ip-10-150-1-55.us-west-2.compute.internal"hostname",:""msgpid":""4759:}ip-10-150-1-55.us-west-2.compute.internal"",
"pid"{:[0] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) WaitQuorum New New ds_transition to Active4759""}msg"
,:""{v"[0] client skip 4 in process jobs because fault"",":v":00msg",,":""name":"name"crucible":","crucible"[2] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) Active Active New ds_transition to WaitActive,"levellevel"",:"30:"v30":0,"name":"crucible",","level"time":":302023-09-22T23:15:04.720394094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped",",,""v"time"::time0"",":name"":"2023-09-22T23:15:04.72039684Z2023-09-22T23:15:04.720403506Z"crucible"",,""level":30,"hostname":"hostname":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",",",pid":"4759pid"}:
4759time":"{}2023-09-22T23:15:04.720434421Z"
","msg"hostname:"":"{ip-10-150-1-55.us-west-2.compute.internal"[2] Transition from New to WaitActive",",""pidv":":04759,,msg""name":":crucible"""",:""[0] Transition from WaitQuorum to Activedownstairs""}
level","{v:"30"msg":":0,"name":"crucible"[0] c96f066c-48e7-437e-b4d0-bca7db257ca7 (777a2755-8455-4e73-a9be-625addff6b55) LiveRepair Faulted Active ds_transition to Faulted",","level"v":0:,",name":"30crucible"","time"level":30:"2023-09-22T23:15:04.720491881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid",":time":"47592023-09-22T23:15:04.720509593Z"},"
,"hostname":{time"":""ip-10-150-1-55.us-west-2.compute.internal"msg2023-09-22T23:15:04.72050618Z"",",:pid":"4759"}
hostname":"{"msg[2] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) Active Active WaitActive ds_transition to WaitQuorumip-10-150-1-55.us-west-2.compute.internal""",:",""v"pid:[0] Transition from LiveRepair to Faulted""0:,,""v":name":"4759crucible"0,,""namelevel":}30":"
crucible","level":30{,"time":"","2023-09-22T23:15:04.72056924Ztime"":"msg",2023-09-22T23:15:04.720575924Z"",hostname"::""hostname":""ip-10-150-1-55.us-west-2.compute.internal","pid":4759ip-10-150-1-55.us-west-2.compute.internal"}[1] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) Active New New ds_transition to WaitActive"
,,""{pid"v"::04759",msg""}:"
RE:0 Bailing with error"name",{":"v":"0crucible","msg"name":",:crucible"","level"level":":3040[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.720635602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":","2023-09-22T23:15:04.720642033Ztime":"","2023-09-22T23:15:04.720635808Zhostname"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid"}:
4759}{
"msg":"{[2] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) Active Active WaitQuorum ds_transition to Active"","msgv":":"0,"name":"[1] Transition from New to WaitActive"crucible",,""levelv""::300,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.720706765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:04.720711694Z""msg":,""hostname":"[2] Transition from WaitQuorum to Active","v":ip-10-150-1-55.us-west-2.compute.internal"0,",pid"":name":"4759crucible","}level"
:30{"msg":","time":"[1] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) Active WaitActive New ds_transition to WaitQuorum","2023-09-22T23:15:04.720756429Z"v",:"0hostname":","name":"crucible"ip-10-150-1-55.us-west-2.compute.internal,"",level"":pid":304759}
{"msg":"740b9b19-2d4a-44eb-b854-682cc860698c is now active with session: d76910b1-bd56-4bb7-8622-025d64ee3acf","v":0,"name":"crucible",",level"":time30":"2023-09-22T23:15:04.720783904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:15:04.72080403Z{","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid"[1] Transition from WaitActive to WaitQuorum":,4759"v"}:
0,"name":{"crucible"","msg":level"":30[0] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.720844643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,}"
time":"2023-09-22T23:15:04.720855065Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) Active WaitQuorum New ds_transition to Active","v"{:0,""namemsg"":":"crucible","level"[0] Transition from Active to Faulted":,"30v":0,"name":"crucible","level":30,"time":","time":2023-09-22T23:15:04.72090176Z"","hostname":"2023-09-22T23:15:04.720897473Z","ip-10-150-1-55.us-west-2.compute.internal"hostname",:""pid":4759}ip-10-150-1-55.us-west-2.compute.internal"
,"pid":4759{}"
msg":"{"[0] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) Faulted Active Active ds_transition to LiveRepairReady"msg,""v"::{"0,"name":"[1] Transition from WaitQuorum to Active"crucible",",msg":"""level"v":Crucible stats registered with UUID: 58683b55-a732-4f77-bcc0-42761edb8654"30:,"0v",:"0name":","crucible"name":",crucible"",,"level""level":time30:":30"2023-09-22T23:15:04.720963282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"{time":"2023-09-22T23:15:04.720978813Z"","msghostname":",""ip-10-150-1-55.us-west-2.compute.internal":,""time"pid":[0] Transition from Faulted to LiveRepairReady"4759:,"}v"
":0,"2023-09-22T23:15:04.72098082Zname":"{","crucible"hostname,"""msg":":"level":30Crucible 58683b55-a732-4f77-bcc0-42761edb8654 has session id: d4b9a97d-da05-496a-91a9-4a4f72a18bebip-10-150-1-55.us-west-2.compute.internal"",,""vpid":0",":name":"4759crucible","level"}:
,30"time":"{2023-09-22T23:15:04.721030266Z",""hostname":"msg":,"ip-10-150-1-55.us-west-2.compute.internal"time":",""pid":2023-09-22T23:15:04.721047527Z"4759,"hostname":"}
ip-10-150-1-55.us-west-2.compute.internal"[2] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) Active Active New ds_transition to WaitActive","pid":{4759,}
""{v"msg""msg"::":"0,"name":"crucible","[0] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) New New New ds_transition to WaitActive"level"[0] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) LiveRepairReady Active Active ds_transition to LiveRepair":,30,""v":v"0:,"0name":,""crucible"name",":level":30","crucible"time":","2023-09-22T23:15:04.721090535Z","hostname":,""time":"2023-09-22T23:15:04.72110221Z","ip-10-150-1-55.us-west-2.compute.internal"hostname":","pid"ip-10-150-1-55.us-west-2.compute.internal",level""::pid"475930:}4759
{}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,",time":""time2023-09-22T23:15:04.721242877Z"":,""hostname":"2023-09-22T23:15:04.721122816Z","ip-10-150-1-55.us-west-2.compute.internal"time",:""pid",2023-09-22T23:15:04.721252478Z:"4759",}"
hostname":hostname"":{"ip-10-150-1-55.us-west-2.compute.internal"",msg"":pid""ip-10-150-1-55.us-west-2.compute.internal":,4759"pid"}:
[0] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) WaitActive New New ds_transition to WaitQuorum4759"{,"v"}:"0msg,"":"name"
[2] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) Active Active WaitActive ds_transition to WaitQuorum:"",crucible""v",:"0level,"":name30":"crucible{","level":30"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":,"0time":",","2023-09-22T23:15:04.721305805Ztime"":,""name":"hostname2023-09-22T23:15:04.721309442Z"crucible":",",""level":hostnameip-10-150-1-55.us-west-2.compute.internal"":,""30pid":4759ip-10-150-1-55.us-west-2.compute.internal",}"
pid":4759}{
"msg":"{"msg"[0] Transition from WaitActive to WaitQuorum:"","v":0,"[2] Transition from WaitActive to WaitQuorum"name",:""v":crucible0",","name"level:"":30crucible","level":30,"time":"2023-09-22T23:15:04.721333266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",,,""time"time:"":""pid2023-09-22T23:15:04.721349377Z"2023-09-22T23:15:04.721351145Z"",,"":hostnamehostname""::""4759}ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:"4759:
4759}
}
{{{"msg":""msg":""msg":"Waiting for Close + ReOpen jobs","v":[2] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) Active Active WaitQuorum ds_transition to Active0"[0] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) WaitQuorum New New ds_transition to Active,,"",v""v"::00,"",name""name":name:"":crucible"","crucible"level,"":level"30:"30crucible","level":30,,""timetime""::""2023-09-22T23:15:04.72141031Z2023-09-22T23:15:04.721411421Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:":47594759}}

{,"{"time":msg""msg""::""2023-09-22T23:15:04.721414435Z","[2] Transition from WaitQuorum to Active[0] Transition from WaitQuorum to Active"",hostname",""vv""::00,",:"name"":"name":crucible""ip-10-150-1-55.us-west-2.compute.internal"crucible,"","levellevel""::3030,"pid":4759}
,,""timetime{""::""2023-09-22T23:15:04.721455737Z2023-09-22T23:15:04.721455885Z""",,""msg":"hostnamehostname""::""RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,"",pidpid""::47594759"}}

{v":{"0"msg"msg:"":","name":"crucible","level":306b6841a1-369e-4ce0-bd40-27f441691315 is now active with session: 604185d3-c867-44d9-80b9-4f85beef9147[1] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) Active New New ds_transition to WaitActive"",,""vv""::00,,""namename""::""cruciblecrucible"",,""levellevel""::3030,,,""time"time:"":""2023-09-22T23:15:04.721505031Z"time",2023-09-22T23:15:04.72150495Z"":,""hostname":hostname"":"2023-09-22T23:15:04.72149803Z"ip-10-150-1-55.us-west-2.compute.internal",ip-10-150-1-55.us-west-2.compute.internal,"",pid""pid:"4759:"}4759
}
{hostname":"{"msg":""msg"ip-10-150-1-55.us-west-2.compute.internal"[1] Transition from New to WaitActive:,"",""pidv""::0[2] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) Active Active Active ds_transition to Faulted,""4759name,"":"v"}:crucible0"
,,""levelname""::30"crucible","level"{:30"msg":"RE:0 close id:1000 queued, notify DS",",v"":time,"":time"":"02023-09-22T23:15:04.721559142Z"2023-09-22T23:15:04.721562392Z",,,"""hostnamename"hostname""::"":"crucible","ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,"level"pid""pid:"4759::}4759
}30
{"{msg":""msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level[1] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) Active WaitActive New ds_transition to WaitQuorum"":30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.721593532Z","hostname,"":time"":,""time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:15:04.721604772Z2023-09-22T23:15:04.721609412Z"",,",""pidhostnamehostname""::""":4759ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:"4759:}}4759
}
{
"msg":"{{[1] Transition from WaitActive to WaitQuorum""msg",:""v":"0,"msgname"":"[2] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) Active Active Faulted ds_transition to LiveRepairReadycrucible"":,,""v":level0",:"30name"":"crucible","RE:0 Wait for result from close command 1000:1level"":30,"v":0,"name":"crucible",,""timelevel":",:""time":302023-09-22T23:15:04.721667239Z"","hostname2023-09-22T23:15:04.721672849Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4759,"pid}"
:4759}
{{"msg":""msg":","[2] Transition from Faulted to LiveRepairReady"time","[1] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) Active WaitQuorum New ds_transition to Activev"",:"0v,""::"name0",:""namecrucible"",:""level":crucible30"2023-09-22T23:15:04.721684457Z",",level"":30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,"4759time":"}
2023-09-22T23:15:04.721712886Z,"",time"":"hostname":"2023-09-22T23:15:04.721717809Z","ip-10-150-1-55.us-west-2.compute.internal"hostname",:""pid":4759}
ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"msg":"{"msg":"[1] Transition from WaitQuorum to Active","v":[2] 6b6841a1-369e-4ce0-bd40-27f441691315 (604185d3-c867-44d9-80b9-4f85beef9147) Active Active LiveRepairReady ds_transition to LiveRepair0",","name"v:"":0crucible,"","name":level"":crucible30","level":30,",time"":time"":"2023-09-22T23:15:04.721754294Z"2023-09-22T23:15:04.721755839Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:":47594759}}

{"{msg":""msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0[2] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) Active Active New ds_transition to WaitActive,"",name""v:"":0crucible,"",name"":"level":crucible"30,"level":30,","time"time:"":"2023-09-22T23:15:04.721788672Z"2023-09-22T23:15:04.721789949Z",","hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid:":47594759}}

{"{msg":""msg":"Waiting for Close + ReOpen jobs","v"[2] Transition from New to WaitActive:"0,,""v"name:"0:","cruciblename"":,""levelcrucible"":,30"level":30,,""timetime""::""2023-09-22T23:15:04.721820711Z2023-09-22T23:15:04.721821803Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47594759}}

{"{msg":""msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible",[2] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) Active Active WaitActive ds_transition to WaitQuorum""level,"":v30":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.721853781Z",","time":hostname"":"2023-09-22T23:15:04.721857688Z",ip-10-150-1-55.us-west-2.compute.internal"","hostname"pid:"":4759}
ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"msg":"{"RE:0 close id:1000 queued, notify DSmsg"":","v":0,"name[2] Transition from WaitActive to WaitQuorum"":","cruciblev"":,0","level"name:":30"crucible","level":30,"time":","time":2023-09-22T23:15:04.721887122Z"","hostname2023-09-22T23:15:04.721889418Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal4759","}
pid":4759}{
"msg":"{"msgRE:0 Wait for result from close command 1000:1"":","v":0,"name":"crucible","level":30[2] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.721918673Z","hostname":","time":ip-10-150-1-55.us-west-2.compute.internal"","pid":2023-09-22T23:15:04.721924363Z4759","}
hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.721961126Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"58683b55-a732-4f77-bcc0-42761edb8654 is now active with session: cac0d111-a31a-4543-a886-1fea695e5af3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.721997243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.722020802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.722039415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.722060146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.722078687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.722121098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.722140246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.722158458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.72217742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.722195734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.722214365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.848260705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.848316724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.848337372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.848355833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:04.848374552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.848393567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:04.848412103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:04.848444206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 71b60e6c-2610-440d-b92b-182648d96e96","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849006184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 71b60e6c-2610-440d-b92b-182648d96e96 has session id: 5838c09f-b248-45c2-9508-0e9bee9f039f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849039755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849079061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849101147Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849122213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849140754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849160955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.84917939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849200207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849218496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849239175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849257745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849278143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849296351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849316952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849335163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849355854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849380413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849402586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849420877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"71b60e6c-2610-440d-b92b-182648d96e96 is now active with session: f64d2a54-56b6-4ceb-9870-e701a34e1d11","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849440937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849461436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849479646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849500193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849518495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849539206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849557666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849575789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849594482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849612618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:04.849631036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.719302752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.719358638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:05.721075103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(1), repair_downstairs: [ClientId(0)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:05.721158849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.721205971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30{,"time":"2023-09-22T23:15:05.721247918Z"","msg"hostname"::""Now ACK the close job"ip-10-150-1-55.us-west-2.compute.internal","pid",:"4759v":,"0":","downstairsname"":"}
crucible","level":30{"msg":"[2] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.721289411Z",","time":"hostname":"2023-09-22T23:15:05.721303975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid"",":pid":47594759}
}
{"msg":"[2] Transition from Active to Faulted","v":0,{"name":"crucible","level":30"msg":"Waiting for 3 jobs (currently 2)","v":0,"name",":time":""crucible2023-09-22T23:15:05.721343652Z"",",hostname":""level":ip-10-150-1-55.us-west-2.compute.internal",30"pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.721369202Z",","hostname":"time":"ip-10-150-1-55.us-west-2.compute.internal","pid":47592023-09-22T23:15:05.721360878Z}"
,"hostname"{:""msg":"Waiting for 3 jobs (currently 2)"ip-10-150-1-55.us-west-2.compute.internal",","v":pid"0:,"4759name":"crucible","}level
":30{","msg"time":":"2023-09-22T23:15:05.721402723Z","hostnameRepair for extent 0 s:0 d:[ClientId(2)]"":","v":0ip-10-150-1-55.us-west-2.compute.internal,",""name":"pidcrucible"":,"4759level"}
:30{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":","2023-09-22T23:15:05.721426971Z"time":","hostname":2023-09-22T23:15:05.721432433Z"","hostname":ip-10-150-1-55.us-west-2.compute.internal"","pid":4759ip-10-150-1-55.us-west-2.compute.internal","pid":,"4759":}
"downstairs"{}
"msg":"{[0] client skip 2 in process jobs because fault"",msg":""v":0,"name"RE:0 Wait for result from repair command 1001:2":,""v":crucible"0,",level":"30name":"crucible","level":30,"time":"2023-09-22T23:15:05.721469903Z",",hostname"":time":""2023-09-22T23:15:05.721471629Z","ip-10-150-1-55.us-west-2.compute.internal"hostname":","pid":4759ip-10-150-1-55.us-west-2.compute.internal,"",":""downstairs"pid"}:
4759}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.721502724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 740b9b19-2d4a-44eb-b854-682cc860698c (d76910b1-bd56-4bb7-8622-025d64ee3acf) LiveRepair Active Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.721525305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.721543631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.721568615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.722517377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.722544592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.722565749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.849594987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.849649238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.849668683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:05.849689485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:2  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done  Done  false
    2    Acked  1001    NoOp       0  Done  Done  Done  false
    3 NotAcked  1002    NoOp       0   New   New   New  false
    4 NotAcked  1003  Reopen       0   New   New   New  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           2      2      2      6
    Sent          0      0      0      0
    Done          2      2      2      6
    Skipped       0      0      0      0
    Error         0      0      0      0
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      2    1
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.591754819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.591809944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.591830294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.591849336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"jobs are: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.591910423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 87064aa7-d0ac-4f60-b676-7a1f3cd25380","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592366562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 87064aa7-d0ac-4f60-b676-7a1f3cd25380 has session id: 08aa62d6-481b-4db0-8b56-7e1bd0f45909","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592402523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592424997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592444151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592475464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592495549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592516223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592534518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592554926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592573194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592593936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592612507Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592632843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592650987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592671435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592689647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592710298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.59272874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592749657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592767892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"87064aa7-d0ac-4f60-b676-7a1f3cd25380 is now active with session: 38df2a71-8b65-46d9-a8e8-79394e623d8a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592787848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.59280819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592831374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592853095Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592871648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 87064aa7-d0ac-4f60-b676-7a1f3cd25380 (38df2a71-8b65-46d9-a8e8-79394e623d8a) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592892373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592910859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592928914Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592947372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592965569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.592983988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.719629198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:06.719731382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:06.719797147Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.719842128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.719878127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.719901965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.719932406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:06.71995231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.719971383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.719990685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 67c26488-dbae-450b-b463-1bdda5022bc7 (70c18185-953d-4518-93af-6f5485ebb3e0) LiveRepair Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.72001214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.720030677Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.720049145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30{"msg":"Now ACK the repair job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.722824235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:15:06.722843295Z"{,"hostname"":"msg":"ip-10-150-1-55.us-west-2.compute.internal","Extent 0 close id:1001 Failed: Error: badpid"":,4759"v":0},"
name":"crucible","level":50{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.722889971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:15:06.722901782Z"{,"hostname":""msg":ip-10-150-1-55.us-west-2.compute.internal"","pid":4759}
RE:0 Wait for result from NoOp command 1002:3","v{":0,""msg":name"":"crucible"RE:0 Wait for result from NoOp command 1002:3",,""v":level"0:,"30name":"crucible","level":30,"time":"2023-09-22T23:15:06.722941861Z",","hostname":"time":"ip-10-150-1-55.us-west-2.compute.internal","pid":47592023-09-22T23:15:06.722940115Z"}
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.723891775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.723937778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:06.85109834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:06.851161671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.851191622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.851212627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.851234856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.851253236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.851271301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:06.851289765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.85130815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.851327504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 71b60e6c-2610-440d-b92b-182648d96e96 (f64d2a54-56b6-4ceb-9870-e701a34e1d11) LiveRepair Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.851349161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.851367532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:06.851385885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.592963515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.593017879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.59304005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72325251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.723304386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.723323993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.723342246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.723361021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.723379523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.723406075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.723425516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done   Err  false
    2    Acked  1001    NoOp       0  Skip  Done  Skip  false
    3    Acked  1002    NoOp       0  Skip  Done  Skip  false
    4    Acked  1003  Reopen       0  Skip  Done  Skip  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          1      4      0      5
    Skipped       3      0      3      6
    Error         0      0      1      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Crucible stats registered with UUID: 98c53138-edfb-4bb6-9deb-ec8ddf307957","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724028133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 98c53138-edfb-4bb6-9deb-ec8ddf307957 has session id: c2840ded-d7e5-484e-b1fc-80b6031c41b9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724063308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724086344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724105188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724126176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724144606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724164909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724183176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724203522Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724260711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72428481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724303426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724323588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724347936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30{"msg":","timeNow move the NoOp job forward"":",2023-09-22T23:15:07.724369905Z""v",":hostname":"0,"name"ip-10-150-1-55.us-west-2.compute.internal",":"pid":crucible"4759,}
"level":{30"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724397904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:07.724393434Z""msg":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":[2] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Active Active WaitActive ds_transition to WaitQuorum4759","v":0},"
name":"crucible","level":30{"msg":"Now ACK the NoOp job","v":0,"name":"crucible",","level":time":"302023-09-22T23:15:07.724428893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,""time":"msg":"2023-09-22T23:15:07.724438774Z",[2] Transition from WaitActive to WaitQuorum"","hostname":v":0","name":"crucible"ip-10-150-1-55.us-west-2.compute.internal",,""pidlevel""::475930}
{"msg":"Finally, move the ReOpen job forward,""time":","v":0,2023-09-22T23:15:07.724463719Z"","namehostname":"":"crucible",ip-10-150-1-55.us-west-2.compute.internal"",level"":30pid":4759}
{"msg":","time":"2023-09-22T23:15:07.724482559Z"[2] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Active Active WaitQuorum ds_transition to Active",,""v"hostname:":"0,"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible",","pidlevel":":304759}
{"msg":"Now ACK the repair job","v":0,","time":"name":"crucible2023-09-22T23:15:07.724503942Z"",","hostname"level":30:"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg,"":"time":"[2] Transition from WaitQuorum to Active2023-09-22T23:15:07.72452059Z"",,""v"hostname":":0,"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible",","pidlevel":30":4759}
{"msg":","time":"RE:0 Wait for result from reopen command 1003:4","v"2023-09-22T23:15:07.724541676Z":,"0,"hostname":"name":"crucible",ip-10-150-1-55.us-west-2.compute.internal"","levelpid":":304759}
{"msg":","time":"98c53138-edfb-4bb6-9deb-ec8ddf307957 is now active with session: 9c9d5503-b2e8-4f16-b53b-34eee863d6b8","2023-09-22T23:15:07.724560813Zv":"0,","hostname":"name":"crucible",ip-10-150-1-55.us-west-2.compute.internal"","level":pid":304759}
,"time":"2023-09-22T23:15:07.724580917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724602408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72462062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724641366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724659776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724680397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724703484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724722529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724741182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724759352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.724777659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_extent_do_repair_all ... ok
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.725327171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 04a09fc3-4f66-480e-84bb-fa73586a5f17","v":0,"name":"crucible","level":30{"msg":"[2] DS Reports error Err(GenericError(\"bad\","time":"2023-09-22T23:15:07.725457844Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"pid":4759},"
v":0,"name":"crucible"{,"level":50"msg":"Crucible 04a09fc3-4f66-480e-84bb-fa73586a5f17 has session id: 3369a2bc-49b1-4d8e-9433-31405f84065e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.725503723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"time":","":"2023-09-22T23:15:07.725515587Z"downstairs","}hostname":"
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""[2] Reports error GenericError(msg\""bad\":"[0] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.725574325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:07.725585649Z","hostname"":msg"":"ip-10-150-1-55.us-west-2.compute.internal"[0] Transition from New to WaitActive","pid",":v"4759:0,,""name"":":"downstairs"crucible"},"
level":30{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":,"30time":"2023-09-22T23:15:07.725630552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{",msg":""time":"2023-09-22T23:15:07.725645931Z","hostname":[0] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) WaitActive New New ds_transition to WaitQuorum"","v":0ip-10-150-1-55.us-west-2.compute.internal",,""name":pid"":crucible4759",",level"""::"30downstairs"}
{"msg":","time":"[2] changed 0 jobs to fault skipped","2023-09-22T23:15:07.725685063Z"v":,"0hostname":","name":"crucible"ip-10-150-1-55.us-west-2.compute.internal",,""level"pid"::475930}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":",crucible"","time":level":"302023-09-22T23:15:07.725713837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
,"time":"2023-09-22T23:15:07.725735588Z",{"hostname":""ip-10-150-1-55.us-west-2.compute.internal"msg",":pid"":4759}
[2] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) LiveRepair Active Active ds_transition to Faulted","v":0{,"name":"crucible"","msg":"level":30[0] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.725784375Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:15:07.725794267Z"pid",":hostname4759":"}ip-10-150-1-55.us-west-2.compute.internal
","pid":4759}
{{"msg"":"msg":"[0] Transition from WaitQuorum to Active","v":0[2] Transition from Active to Faulted","name":","crucible"v",":level"0:30,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.725836172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) Active New New ds_transition to WaitActive","v":0,","time":"name":"crucible","2023-09-22T23:15:07.725840217Z"level":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759},"
time":"2023-09-22T23:15:07.72586166Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"msg":"{"msg":"RE:0 Wait for result from reopen command 1003:4"[1] Transition from New to WaitActive",,""vv""::0,"0name":","crucible"name,""level":30:"crucible","level":30,"time":"2023-09-22T23:15:07.725896029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) Active WaitActive New ds_transition to WaitQuorum,"","v":0time",":name":""crucible","level":302023-09-22T23:15:07.725899713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"time":}"
2023-09-22T23:15:07.72592336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759{}
"{msg":""msg":"[1] Transition from WaitActive to WaitQuorumExtent 0 close id:1003 Failed: Error: bad"",","v":0,"vname":""crucible":,"0level":,"30name":"crucible","level":50,"time":"2023-09-22T23:15:07.725958484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) Active WaitQuorum New ds_transition to Active","v":0,",name":""crucible",time":""level":302023-09-22T23:15:07.725962201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,"4759time":"}2023-09-22T23:15:07.725983135Z
","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759{}
"msg"{:""msg":"[0] client skip 4 in process jobs because fault"[1] Transition from WaitQuorum to Active",","v":v"0:,"0name":,""namecrucible"":","crucible"level":,30"level":30,"time":"2023-09-22T23:15:07.726021675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","[2] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) Active Active New ds_transition to WaitActive"time",:""v":0,"name":"2023-09-22T23:15:07.726023634Zcrucible"","level":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,","time":"":"2023-09-22T23:15:07.726047685Zdownstairs"","}hostname":"
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""[2] Transition from New to WaitActivemsg"":,""v":0,"name":"crucible"[0] changed 0 jobs to fault skipped","level,":30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.726081863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.726087987Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:15:07.726102728Z"pid,""hostname":":4759ip-10-150-1-55.us-west-2.compute.internal","pid,"":"4759:"}
downstairs"}{
"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible"{,"level":30"msg":","time":"[0] 58683b55-a732-4f77-bcc0-42761edb8654 (cac0d111-a31a-4543-a886-1fea695e5af3) LiveRepair Active Faulted ds_transition to Faulted"2023-09-22T23:15:07.72613854Z",,""vhostname"":":0ip-10-150-1-55.us-west-2.compute.internal",,""pidname":"":4759crucible"}
,"level"{:30"msg":"[2] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.726170694Z","hostname":",ip-10-150-1-55.us-west-2.compute.internal"","time"pid"::4759"}
2023-09-22T23:15:07.726164127Z"{,""msg":"hostname":"[2] Transition from WaitQuorum to Active","v":0,"ip-10-150-1-55.us-west-2.compute.internal"name",:""cruciblepid"":,"4759level":30}
,"time":"2023-09-22T23:15:07.726200961Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
"msg"{:""msg":"[0] Transition from LiveRepair to Faulted","v":04a09fc3-4f66-480e-84bb-fa73586a5f17 is now active with session: b0b519cf-95b7-4a43-b79b-312a989a1592"0,","vname":":"0,"cruciblename"":","crucible","level"level"::3030,"time":"2023-09-22T23:15:07.726232678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":[1] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) Active Active Active ds_transition to Faulted"","v":0,"name":"2023-09-22T23:15:07.726233471Zcrucible"","level":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759},"
time":"2023-09-22T23:15:07.72625829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4759}
"msg{":""msg":"RE:0 Bailing with error"[1] Transition from Active to Faulted",,""v":0v",":name"0:",crucible"","name"level":30:"crucible","level":40,"time":"2023-09-22T23:15:07.726293115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) Active Faulted Active ds_transition to LiveRepairReady","v":0,,""name":time""crucible":,""level":302023-09-22T23:15:07.726296494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"time":"}2023-09-22T23:15:07.72631923Z"
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72634437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 04a09fc3-4f66-480e-84bb-fa73586a5f17 (b0b519cf-95b7-4a43-b79b-312a989a1592) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.726365056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.726383563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_io_above_el_skipped ... ok
{"msg":"Crucible stats registered with UUID: 73140564-9a43-4886-9974-6360c31b2d26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.726758749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 73140564-9a43-4886-9974-6360c31b2d26 has session id: 1dc8588b-8a0c-4272-939c-6e839c534254","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.726811816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72685011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.726883055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.726927451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.726962151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72699845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.727030291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.727069101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.727101112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"{"[1] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Active WaitActive New ds_transition to WaitQuorum"msg",:""v":0,"Crucible stats registered with UUID: f25cb6f0-d847-4044-b2ba-64db5febf8b4"name":","crucible"v":,"0level,""name":":crucible"30,"level":30,"time":"2023-09-22T23:15:07.727146463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"}time":"
2023-09-22T23:15:07.727148512Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal"","msg"pid"::"4759}
[1] Transition from WaitActive to WaitQuorum","v":0,{"name":""crucible"msg":","level":30Crucible f25cb6f0-d847-4044-b2ba-64db5febf8b4 has session id: 5b21f270-e95e-487f-a9e3-73e37056f04d","v":0,"name":"crucible","level":30,"time":","time"2023-09-22T23:15:07.727193284Z":","2023-09-22T23:15:07.727197716Z"hostname",":hostname"":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid":"4759,"}pid
":4759{}
"msg":"{"msg":"[0] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30[1] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible,""time":","2023-09-22T23:15:07.727233133Z"level",":hostname30":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":,""timecrucible"":,""level":302023-09-22T23:15:07.727246991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,"4759time":"}
2023-09-22T23:15:07.727262726Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal",""pid"msg:"4759:"}
[1] Transition from WaitQuorum to Active"{,"v""msg":":0,"name":"crucible","[0] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) WaitActive New New ds_transition to WaitQuorum"level,"":v":030,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.727298985Z",,""hostname":time"":"ip-10-150-1-55.us-west-2.compute.internal","pid"2023-09-22T23:15:07.727296987Z":4759,"}
hostname":"{"msg":"ip-10-150-1-55.us-west-2.compute.internal",[0] Transition from WaitActive to WaitQuorum"","pidv":"0:,"4759name":"}
crucible","level":30{"msg":","time":"2023-09-22T23:15:07.727332158Z","hostname[2] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Active Active New ds_transition to WaitActive"":","v":0ip-10-150-1-55.us-west-2.compute.internal",","name"pid"::4759"}
crucible",{"level"":msg30":"[0] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:15:07.727356259Z"time":,""2023-09-22T23:15:07.7273626Z"hostname",":hostname":""ip-10-150-1-55.us-west-2.compute.internal","pid":4759ip-10-150-1-55.us-west-2.compute.internal}"
,"pid":{4759}"
msg":"[0] Transition from WaitQuorum to Active"{,"v":0,"name"":"msg"crucible":,""level":30[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.727403989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":"[1] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) Active New New ds_transition to WaitActive"2023-09-22T23:15:07.727411123Z","v":0,,""name":"hostnamecrucible"",":level":"30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:07.727434521Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Active Active WaitActive ds_transition to WaitQuorum"[1] Transition from New to WaitActive",",v"":v":00,",name":""crucible"name",":"level":30crucible","level":30,"time":"2023-09-22T23:15:07.727465462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time{":""msg":"2023-09-22T23:15:07.727467682Z","hostname":"[1] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) Active WaitActive New ds_transition to WaitQuorum","v":0,"ip-10-150-1-55.us-west-2.compute.internal"name,":""pidcrucible"",":level":304759}
{,"time":""msg":"2023-09-22T23:15:07.727494222Z","hostname":"[2] Transition from WaitActive to WaitQuorum"ip-10-150-1-55.us-west-2.compute.internal",,""pidv""::47590}
,"name{":""msgcrucible"":,""level":[1] Transition from WaitActive to WaitQuorum30","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:15:07.72752422Z"time",":hostname":""ip-10-150-1-55.us-west-2.compute.internal","pid"2023-09-22T23:15:07.727521341Z":4759,"}
hostname":"{"msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}[1] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) Active WaitQuorum New ds_transition to Active"
,"v":0,"name":"{crucible","level":30"msg":","time":"[2] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Active Active WaitQuorum ds_transition to Active"2023-09-22T23:15:07.727555455Z,"","v"hostname":":0,"ip-10-150-1-55.us-west-2.compute.internal","namepid":"4759:"}
crucible","{level":"30msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,,""time":time"":"2023-09-22T23:15:07.727584886Z","hostname":"2023-09-22T23:15:07.72758053Z","ip-10-150-1-55.us-west-2.compute.internal","hostname"pid:":"4759}
ip-10-150-1-55.us-west-2.compute.internal","{pid":"4759msg":"}
[2] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) Active Active New ds_transition to WaitActive","v":0,"{name":"crucible",""level":msg"30:"[2] Transition from WaitQuorum to Active","v":0,"name":","crucibletime":"","2023-09-22T23:15:07.727617799Z"level",":hostname":30"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible",","level":30time":"2023-09-22T23:15:07.72763104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal,""time":,""pid"2023-09-22T23:15:07.727643074Z":,"4759hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}{
"msg"{:""msg":"[2] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) Active Active WaitActive ds_transition to WaitQuorum73140564-9a43-4886-9974-6360c31b2d26 is now active with session: 521e1392-21bb-441e-9084-601140ec5551"",,""vv""::00,","name"name":":crucible"","level":30crucible","level":30,"time":"2023-09-22T23:15:07.727684506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time{":""msg":"2023-09-22T23:15:07.727686836Z"[2] Transition from WaitActive to WaitQuorum",,""v":0hostname",":name":""crucible","level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:07.727709683Z","hostname"":msg"":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Active Active Active ds_transition to Faulted","{v":0",msg":""name":"crucible","level":[2] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) Active Active WaitQuorum ds_transition to Active"30,"v":0,"name":"crucible","level":30,"time,":""time"2023-09-22T23:15:07.727741918Z":,""hostname":"2023-09-22T23:15:07.727739037Z"ip-10-150-1-55.us-west-2.compute.internal",,""pid":4759hostname"}
:"{"msg"ip-10-150-1-55.us-west-2.compute.internal":","pid":[2] Transition from WaitQuorum to Active"4759,"v":0},"
name":"crucible","level":30{"msg":"[1] Transition from Active to Faulted","v":0,",time":""name":"2023-09-22T23:15:07.727777087Z","crucible"hostname":","level"ip-10-150-1-55.us-west-2.compute.internal":,"30pid":4759}
{"msg":"f25cb6f0-d847-4044-b2ba-64db5febf8b4 is now active with session: 3ec1c360-ac3c-465c-a512-4327d950daa8","v":0,"name":"crucible",,""level":time"30:"2023-09-22T23:15:07.727795582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",",time"":"pid":47592023-09-22T23:15:07.727809116Z","}hostname
":"ip-10-150-1-55.us-west-2.compute.internal","{pid":4759}
"msg":"{"msg":"[1] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) Active Active Active ds_transition to Faulted","v":0,"[1] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Active Faulted Active ds_transition to LiveRepairReady"name":","crucible"v":,0","level":30name":"crucible","level":30,"time":"2023-09-22T23:15:07.727845154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,{"time"":"msg":"[1] Transition from Active to Faulted"2023-09-22T23:15:07.727848551Z,""v":0,",name":""crucible","hostnamelevel":30":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:07.727873047Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
[1] Transition from Faulted to LiveRepairReady"{,"v"":msg":0","name":"crucible","level":[1] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) Active Faulted Active ds_transition to LiveRepairReady"30,"v":0,"name":"crucible","level":30,"time":","2023-09-22T23:15:07.727903237Z"time,""hostname":":"ip-10-150-1-55.us-west-2.compute.internal","pid":2023-09-22T23:15:07.727900662Z4759"}
,"{hostname":""msg":"[1] Transition from Faulted to LiveRepairReady"ip-10-150-1-55.us-west-2.compute.internal",,""v":pid"0:,"4759name":"}crucible"
,"level":30{"msg":","time":"2023-09-22T23:15:07.727937863Z","hostname":"[1] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Active LiveRepairReady Active ds_transition to LiveRepair"ip-10-150-1-55.us-west-2.compute.internal",",pid":"4759v":}0
,"name{":""msg":"crucible","level":30[1] f25cb6f0-d847-4044-b2ba-64db5febf8b4 (3ec1c360-ac3c-465c-a512-4327d950daa8) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,,""time":"time":"2023-09-22T23:15:07.727966384Z","hostname":"2023-09-22T23:15:07.727962103Z","ip-10-150-1-55.us-west-2.compute.internal",hostname""pid"::4759"}
ip-10-150-1-55.us-west-2.compute.internal",{"pid""msg":":4759}[1] Transition from LiveRepairReady to LiveRepair"
,"v":0,"name":"{crucible","level":30"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,","nametime":"":"2023-09-22T23:15:07.727997453Z"crucible",",hostname":""level":ip-10-150-1-55.us-west-2.compute.internal"30,"pid":4759}
{"msg":"Write to Extent 0:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.728013018Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:15:07.728022237Z",,""hostnamepid"":":4759ip-10-150-1-55.us-west-2.compute.internal","}pid
{""msg":":4759Waiting for Close + ReOpen jobs"}
,"v":0,"name{":"crucible"","msg":level"":30Write to Extent 0:0:9 under repair","v":0,"name":"crucible","level":40,,""timetime""::""2023-09-22T23:15:07.728142241Z2023-09-22T23:15:07.728147415Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:":47594759}
}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728198889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728236966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728268973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_io_at_el_sent ... ok
{"msg":"Crucible stats registered with UUID: 1bd3598e-c0ae-461f-b737-8e54c980aeab","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728786169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 1bd3598e-c0ae-461f-b737-8e54c980aeab has session id: be7e3185-beed-4c0b-bc34-ecea7cde1037","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72881992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728842359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728861352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728882172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72890084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728921301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728939602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728959764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72897807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.728998746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729017146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729037658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729055724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729076117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729100244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729122126Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729140688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729161294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729179498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"1bd3598e-c0ae-461f-b737-8e54c980aeab is now active with session: c23143a7-8706-4bb2-a232-a549da748e27","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729199212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72921958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729237768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72925812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729276451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1bd3598e-c0ae-461f-b737-8e54c980aeab (c23143a7-8706-4bb2-a232-a549da748e27) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.72929721Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729315784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_io_below_el_sent ... ok
{"msg":"Crucible stats registered with UUID: 630c7f91-b4d8-4905-b080-0d171cd9028f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729910495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 630c7f91-b4d8-4905-b080-0d171cd9028f has session id: e3024dc0-53d3-4056-83d8-9a8fb9a977f8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729943952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729965843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.729984701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730010902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730030793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730051354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730069438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.7300897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730108199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730128792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730147276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73016771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730186037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730206287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730224241Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730244557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730262728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730283261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730301529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"630c7f91-b4d8-4905-b080-0d171cd9028f is now active with session: 71ef5e0f-f803-43f6-b820-271fc47e3485","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730321394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730341655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730363605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730385074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730403751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 630c7f91-b4d8-4905-b080-0d171cd9028f (71ef5e0f-f803-43f6-b820-271fc47e3485) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730424247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.730442761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_io_no_el_skipped ... ok
{"msg":"Crucible stats registered with UUID: 6bffdf01-5cd2-4635-97e7-762d833316f9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731029866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 6bffdf01-5cd2-4635-97e7-762d833316f9 has session id: 09cb16a5-bde7-4030-81f3-d987e91d8bcf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731063183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731085229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731103995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731124797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731142993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731163289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731181381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731201865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731220032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73124041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731258739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731285306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731304856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731325504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731343581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731364058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731382303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731402675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731420949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"6bffdf01-5cd2-4635-97e7-762d833316f9 is now active with session: 48be3975-38f5-4ada-a823-b521356c3118","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73144066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731461108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731479096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731499521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731517927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6bffdf01-5cd2-4635-97e7-762d833316f9 (48be3975-38f5-4ada-a823-b521356c3118) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731538418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.731556991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 1:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731575069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 1:1:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731593063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 1:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731615676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:0:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731634676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:1:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.73165299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:2:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731671133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"IO Write 1004 on eur 1 Added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.73168935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Create read repair deps for extent 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.73170753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"IO Read  1005 extent 1 added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731725388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 1:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731743385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 1:1:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731761286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 1:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731779079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:0:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731797345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:1:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731815494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:2:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731833792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"IO Write 1006 on eur 1 Added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.731851932Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_io_span_el_sent ... ok
{"msg":"Crucible stats registered with UUID: cde6561c-b94d-442b-83c1-194a7b340204","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732436301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible cde6561c-b94d-442b-83c1-194a7b340204 has session id: 99f5768b-a5f4-486b-8bbc-25f1561f34f6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73246991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732491833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732510651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732531399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732549783Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732575739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73259534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732615802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732633961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732654652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732672882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732693181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732711269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73273171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732749781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732770162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732788448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732808985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732827247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"cde6561c-b94d-442b-83c1-194a7b340204 is now active with session: 70ef231b-de88-4e90-a397-be55ff1dea6d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732847036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732867319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732885508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732905883Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732928593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cde6561c-b94d-442b-83c1-194a7b340204 (70ef231b-de88-4e90-a397-be55ff1dea6d) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73295034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.732969207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Create read repair deps for extent 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.732987401Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"IO Read  1004 extent 1 added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.733005469Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_read_span_el_sent ... ok
{"msg":"Crucible stats registered with UUID: ac10043e-f771-4b20-9401-d688ee9fd44c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733610911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ac10043e-f771-4b20-9401-d688ee9fd44c has session id: 20e39922-f402-4d04-b0fd-13c2207ef5b6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733644495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733666408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733685503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733706173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733724802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73374549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733764139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733784421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733802629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733823325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733841719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733868053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733887555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733908019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733926261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73394676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733965221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.733985621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734003944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"ac10043e-f771-4b20-9401-d688ee9fd44c is now active with session: ed0be356-db20-44b7-8af6-f9591507659c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734024278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734045015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734063294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734083869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734102364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac10043e-f771-4b20-9401-d688ee9fd44c (ed0be356-db20-44b7-8af6-f9591507659c) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734123117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734141674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 1:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.734159793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 1:1:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.734177841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 1:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.734196021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:0:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.734218485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:1:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.734237686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:2:9 past extent under repair 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.734255969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"IO Write 1004 on eur 1 Added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.734274285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_write_span_el_sent ... ok
{"msg":"Crucible stats registered with UUID: 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734884502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e has session id: 227380e3-74cf-458b-8f99-cd6384b7ccf3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734917681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734939399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734958184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734978753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.734997189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735017529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735035909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735056462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735074578Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735095151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735113425Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735133588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73515177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735177976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735197435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735218291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735236387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735256733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735274911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"7f6c56b0-84fc-4f4e-ad81-9acd25765e9e is now active with session: e660d75e-dc92-48f6-9e7d-44cbb6b32053","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73529471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73531512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735333192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735353664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735371999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 7f6c56b0-84fc-4f4e-ad81-9acd25765e9e (e660d75e-dc92-48f6-9e7d-44cbb6b32053) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735392579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.735411036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 0:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.735429021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 0:1:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.735447103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 0:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.735465174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 1:0:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.7354833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 1:1:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.73550162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 1:2:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.735523485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:0:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.735542538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:1:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.735561024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 2:2:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.735579129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"IO Write 1008 on eur 0 Added deps 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.7355972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"IO Write 1008 on eur 0 Added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.735615366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_write_span_two_el_sent ... ok
{"msg":"Crucible stats registered with UUID: 797301b9-b7b2-43e8-990e-87a8ef0b3065","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736220936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 797301b9-b7b2-43e8-990e-87a8ef0b3065 has session id: a65dfd88-a5bf-42eb-ac78-f30129ced51d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73625557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736278058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736297213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736318032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736336492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736357008Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73637521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736395763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736414042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73643479Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736453301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736473768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73649806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736520406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736538927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736559588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736578073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736598784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736617081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"797301b9-b7b2-43e8-990e-87a8ef0b3065 is now active with session: 01780b64-54c6-47e3-9047-57d69957a1c2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736637109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736657678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736676103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736696809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.73671537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 797301b9-b7b2-43e8-990e-87a8ef0b3065 (01780b64-54c6-47e3-9047-57d69957a1c2) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736736338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.736754998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_reserve_extent_repair_ids ... ok
Testing repair with s:0 r:[ClientId(1)]
Sep 22 23:15:07.737 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.737 INFO Repair for extent 0 s:0 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:0 r:[ClientId(2)]
Sep 22 23:15:07.737 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.737 INFO Repair for extent 0 s:0 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0)]
Sep 22 23:15:07.737 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.737 INFO Repair for extent 0 s:1 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(2)]
Sep 22 23:15:07.737 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.737 INFO Repair for extent 0 s:1 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0)]
Sep 22 23:15:07.737 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.737 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(1)]
Sep 22 23:15:07.737 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.737 INFO Repair for extent 0 s:2 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_dirty_needs_repair_one ... ok
Testing repair with s:0 r:[ClientId(1), ClientId(2)]
Sep 22 23:15:07.737 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.737 INFO Repair for extent 0 s:0 d:[ClientId(1), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0), ClientId(2)]
Sep 22 23:15:07.737 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.737 INFO Repair for extent 0 s:1 d:[ClientId(0), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0), ClientId(1)]
Sep 22 23:15:07.737 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.737 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.737 INFO Repair for extent 0 s:2 d:[ClientId(0), ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_dirty_needs_repair_two ... ok
Testing repair with s:0 r:[ClientId(1)]
Sep 22 23:15:07.738 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.738 INFO Repair for extent 0 s:0 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:0 r:[ClientId(2)]
Sep 22 23:15:07.738 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.738 INFO Repair for extent 0 s:0 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0)]
Sep 22 23:15:07.738 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.738 INFO Repair for extent 0 s:1 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(2)]
Sep 22 23:15:07.738 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.738 INFO Repair for extent 0 s:1 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0)]
Sep 22 23:15:07.738 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.738 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(1)]
Sep 22 23:15:07.738 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.738 INFO Repair for extent 0 s:2 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_flush_higher_needs_repair_one ... ok
Testing repair with s:0 r:[ClientId(1), ClientId(2)]
Sep 22 23:15:07.738 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.738 INFO Repair for extent 0 s:0 d:[ClientId(1), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0), ClientId(2)]
Sep 22 23:15:07.738 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.738 INFO Repair for extent 0 s:1 d:[ClientId(0), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0), ClientId(1)]
Sep 22 23:15:07.738 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.738 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.738 INFO Repair for extent 0 s:2 d:[ClientId(0), ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_flush_higher_needs_repair_two ... ok
Testing repair with s:0 r:[ClientId(1)]
Sep 22 23:15:07.739 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.739 INFO Repair for extent 0 s:0 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:0 r:[ClientId(2)]
Sep 22 23:15:07.739 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.739 INFO Repair for extent 0 s:0 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0)]
Sep 22 23:15:07.739 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.739 INFO Repair for extent 0 s:1 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(2)]
Sep 22 23:15:07.739 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.739 INFO Repair for extent 0 s:1 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0)]
Sep 22 23:15:07.739 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.739 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(1)]
Sep 22 23:15:07.739 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.739 INFO Repair for extent 0 s:2 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_flush_lower_needs_repair_one ... ok
Testing repair with s:0 r:[ClientId(1), ClientId(2)]
Sep 22 23:15:07.739 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.739 INFO Repair for extent 0 s:0 d:[ClientId(1), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0), ClientId(2)]
Sep 22 23:15:07.739 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.739 INFO Repair for extent 0 s:1 d:[ClientId(0), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0), ClientId(1)]
Sep 22 23:15:07.739 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.739 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.740 INFO Repair for extent 0 s:2 d:[ClientId(0), ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_flush_lower_needs_repair_two ... ok
Testing repair with s:0 r:[ClientId(1)]
Sep 22 23:15:07.740 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.740 INFO Repair for extent 0 s:0 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:0 r:[ClientId(2)]
Sep 22 23:15:07.740 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.740 INFO Repair for extent 0 s:0 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0)]
Sep 22 23:15:07.740 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.740 INFO Repair for extent 0 s:1 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(2)]
Sep 22 23:15:07.740 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.740 INFO Repair for extent 0 s:1 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0)]
Sep 22 23:15:07.740 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.740 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(1)]
Sep 22 23:15:07.740 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.740 INFO Repair for extent 0 s:2 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_gen_higher_needs_repair_one ... ok
Testing repair with s:0 r:[ClientId(1), ClientId(2)]
Sep 22 23:15:07.740 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.740 INFO Repair for extent 0 s:0 d:[ClientId(1), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0), ClientId(2)]
Sep 22 23:15:07.740 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.740 INFO Repair for extent 0 s:1 d:[ClientId(0), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0), ClientId(1)]
Sep 22 23:15:07.740 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.740 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.740 INFO Repair for extent 0 s:2 d:[ClientId(0), ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_gen_higher_needs_repair_two ... ok
Testing repair with s:0 r:[ClientId(1)]
Sep 22 23:15:07.741 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.741 INFO Repair for extent 0 s:0 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:0 r:[ClientId(2)]
Sep 22 23:15:07.741 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.741 INFO Repair for extent 0 s:0 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0)]
Sep 22 23:15:07.741 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.741 INFO Repair for extent 0 s:1 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(2)]
Sep 22 23:15:07.741 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.741 INFO Repair for extent 0 s:1 d:[ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0)]
Sep 22 23:15:07.741 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.741 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(1)]
Sep 22 23:15:07.741 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.741 INFO Repair for extent 0 s:2 d:[ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_gen_lower_needs_repair_one ... ok
Testing repair with s:0 r:[ClientId(1), ClientId(2)]
Sep 22 23:15:07.741 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.741 INFO Repair for extent 0 s:0 d:[ClientId(1), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(0), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:1 r:[ClientId(0), ClientId(2)]
Sep 22 23:15:07.741 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.741 INFO Repair for extent 0 s:1 d:[ClientId(0), ClientId(2)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(1), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(2)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Testing repair with s:2 r:[ClientId(0), ClientId(1)]
Sep 22 23:15:07.741 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.741 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.741 INFO Repair for extent 0 s:2 d:[ClientId(0), ClientId(1)], : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveRepair { dependencies: [], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:1234, repair_downstairs: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
test live_repair::repair_test::test_solver_gen_lower_needs_repair_two ... ok
Sep 22 23:15:07.742 DEBG Get repair info for 0 source, : downstairs
Sep 22 23:15:07.742 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.742 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.742 INFO No repair needed for extent 0, : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveNoOp { dependencies: [] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Passed for source 0
Sep 22 23:15:07.742 DEBG Get repair info for 1 source, : downstairs
Sep 22 23:15:07.742 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.742 DEBG Get repair info for 2 bad, : downstairs
Sep 22 23:15:07.742 INFO No repair needed for extent 0, : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveNoOp { dependencies: [] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Passed for source 1
Sep 22 23:15:07.742 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:15:07.742 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:15:07.742 DEBG Get repair info for 1 bad, : downstairs
Sep 22 23:15:07.742 INFO No repair needed for extent 0, : downstairs
repair op: DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentLiveNoOp { dependencies: [] }, state: ClientData([New, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }
Passed for source 2
test live_repair::repair_test::test_solver_no_work ... ok
{"msg":"Crucible stats registered with UUID: 6909e08a-657e-411e-b860-976a08df7f3b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.742904779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 6909e08a-657e-411e-b860-976a08df7f3b has session id: 76d7d52d-b000-4099-8d66-fc470c98c52f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.742939075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.742961076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.742980187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743000767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743019369Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743039886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743058005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743078068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743102375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743124564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743143087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743163449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743181616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743201885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743220007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743240502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743258866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743279257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743297317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"6909e08a-657e-411e-b860-976a08df7f3b is now active with session: 9af93bfb-1d45-4033-8c5d-51c86e95e775","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74331693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743337546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743355596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743376261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743394695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74341521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743437866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743459489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743478153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743498671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743516969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 6909e08a-657e-411e-b860-976a08df7f3b (9af93bfb-1d45-4033-8c5d-51c86e95e775) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743537744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.743556267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write to Extent 0:2:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.743574376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Write 1:0:9 past extent under repair 0","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.743592635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"IO Write 1005 on eur 0 Added deps 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.743610777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Create read repair deps for extent 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.743628893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"IO Read  1010 extent 0 added deps 2","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.743646842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_spicy_live_repair ... ok
test mend::test::reconcile_dirty_length_bad - should panic ... ok
{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.744734939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.744766374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:2  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74478685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:2  flush: 2 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.744807632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.744827048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"find dest for source 2 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.744846615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.744871991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.744892739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
test mend::test::reconcile_dirty_mismatch_c0 ... ok
{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.745395538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.745427328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:2  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.745447564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:2  flush: 2 2 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.745467998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.745488072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:2  dirty: false true false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.745507852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"find dest for source 1 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.745527578Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 1, add dest 0 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.745546924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 1, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.745566077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
test mend::test::reconcile_dirty_mismatch_c1 ... ok
{"msg":"Extents 1 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.745988697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746021164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:1  gens: 8 8 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746041255Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:1  flush: 1 1 1 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746061365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74608109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:1  dirty: false false true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746100528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746120392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746140169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746159348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
test mend::test::reconcile_dirty_mismatch_c2 ... ok
{"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746622952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746654712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:0  gens: 9 9 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746674728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:0  flush: 2 2 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746715586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:0  dirty: true true true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746734814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746754524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746773888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746792988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746812648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746831091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:3  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746850354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:3  flush: 1 1 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746870694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74689058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:3  dirty: true true true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746909858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746929141Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746948203Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.746967573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
ef.dest [
    ClientId(
        1,
    ),
    ClientId(
        2,
    ),
]
test mend::test::reconcile_dirty_true ... ok
{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747471453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747502631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74752904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  flush: 2 1 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747551152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747571034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747590744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747610274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747629537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747649838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747668379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747687686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  flush: 3 1 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747708006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747727374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747747046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747766405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747785695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747805422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747823807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747843193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  flush: 1 2 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747863522Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747882962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747907101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747927477Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747947213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74796715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.747985452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:4  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748004891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:4  flush: 2 2 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748025503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748045585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:4  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748065094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.7480852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748105477Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748124878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 5 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748144721Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748163068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:5  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748185272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:5  flush: 3 2 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748206396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748225998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748245579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748264863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748284129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 6 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74830859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748327933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:6  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748347679Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:6  flush: 1 3 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748368263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748387888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748408055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748427461Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748446867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 7 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748466836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748485091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:7  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748504742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:7  flush: 2 3 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748525268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748544823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748564738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748584125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748603511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 8 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748623339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748641872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:8  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748661612Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:8  flush: 3 3 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748682055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748706796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:8  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748727413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748747581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748767471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.748786934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_a ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749276195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749307901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749328121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 1 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749348753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749368344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749388069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74940729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74942664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749446852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74946523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749484547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  flush: 2 1 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749504994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749524574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74954394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749569645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749590883Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749610257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749630052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749648311Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749667492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  flush: 3 1 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749687659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749706996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749726647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749745778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749764907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749784564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74980254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749821961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  flush: 1 2 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749842178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749861996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749881367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749901243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749920947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749944069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 5 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.74996492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.749983376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:5  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750002738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:5  flush: 3 2 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750023412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750042881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750062541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750081713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750100888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 6 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750120705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750138838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:6  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750157948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:6  flush: 1 3 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750178187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750197585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750217258Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750236463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75025549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 7 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750275189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75029327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:7  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750312544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:7  flush: 2 3 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750336963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75035728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750377113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750396392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750415598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 8 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750435303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75045339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:8  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750472566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:8  flush: 3 3 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750492815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750512539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:8  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750531841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750551664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750571296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.750590513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_b ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751095623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751127059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751147663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 1 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751168164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751187737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751213283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751234084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75125364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751274019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751292648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751311974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  flush: 2 1 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751332349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75135194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751371768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751391074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751410311Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751430073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751448431Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751467907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  flush: 3 1 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751488248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751507979Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751527478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751547354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751567144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751586695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751610474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751629953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751649537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  flush: 1 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751669936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751689489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751709202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751728536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751747805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751767592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751785901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:4  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751805121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:4  flush: 2 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751825511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751844925Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751864756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751884087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751903355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 5 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751922939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751941359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:5  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75196062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:5  flush: 3 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.751980949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752004713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:5  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752025599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752045873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75206568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75208495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 6 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752104853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752123156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:6  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752142332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:6  flush: 1 3 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752162551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752182415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:6  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752201962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752221749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752241545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752260683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 7 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752280374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752298555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:7  gens: 1 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752317772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:7  flush: 2 3 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752338111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752358057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:7  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752381637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752402584Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752422649Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.752441945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_c ... ok
test mend::test::reconcile_flush_length_bad - should panic ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753338158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753369848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 9 9 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753390292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 2 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753411025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753431142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753450763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753470748Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753490606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753509984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_mismatch_c0 ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753954285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.753985665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 9 9 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754005964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754026532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754046173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754065935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754090937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75411163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754132207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754150811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  gens: 8 8 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754170326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  flush: 2 1 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754190652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754210131Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754229889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754249064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754268179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754287935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754306271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754325582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  flush: 3 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754345847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754365794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754385558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75440572Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754425463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754444756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75446839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754487739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754507241Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  flush: 3 2 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754527686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754547069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754566922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754586235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 0, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754605481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 4 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754625304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754643557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:4  gens: 6 6 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754663037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:4  flush: 1 3 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754683269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754702878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:4  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754722222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754742169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754761792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754780949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 5 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75480066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754818825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:5  gens: 5 5 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754838222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:5  flush: 2 3 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754863016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754883454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754903243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754922557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.754941638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_mismatch_c1 ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755413105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755445366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 9 9 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755465822Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 1 2 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755486336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755506026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75552571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755545068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755564246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755584502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755602951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755622316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  flush: 1 1 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755642618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755662016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755681735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755707092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.755728223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
test mend::test::reconcile_flush_mismatch_c2 ... ok
{"msg":"generation number mismatch 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756207636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756238943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:1  gens: 2 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756259329Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756279227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756298547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756317637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756337573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756356124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:2  gens: 3 1 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756375172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756394763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756413876Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756432851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756452473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756470969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:3  gens: 1 2 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756490354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756509861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756528745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756547784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756572833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756592297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:4  gens: 2 2 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756611599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:4  flush: 1 1 1 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756631579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756651316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:4  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756670709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75669051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756710359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75672947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756748812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756767043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:5  gens: 3 2 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756786102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756805731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756824786Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756843794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75686319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756881539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:6  gens: 1 3 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756900506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756920189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756939273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756958153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.756981698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757000865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:7  gens: 2 3 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757020356Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757040228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757059411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757078614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757098193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757116371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:8  gens: 3 3 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757135633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:8  flush: 1 1 1 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757155387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75717516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:8  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757194456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757214298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75723394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757253094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
test mend::test::reconcile_gen_a ... ok
{"msg":"generation number mismatch 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757749679Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757781653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:0  gens: 1 1 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757801994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75782185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757841173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757865957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757887078Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757905533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:1  gens: 2 1 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757924933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:1  flush: 1 1 1 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757945195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757964982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:1  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.757984297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758004226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758024023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75804318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758062785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758080926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:2  gens: 3 1 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758100382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758120134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758139349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758158562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758178081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758199732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:3  gens: 1 2 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758219281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:3  flush: 1 1 1 scs: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758239314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758263391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:3  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758284167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758304212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758323949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758343252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758363013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758381207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:5  gens: 3 2 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758400567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758420273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758439515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758458658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75847817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758496472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:6  gens: 1 3 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758515587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758535417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758554705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758574028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758593773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758612166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:7  gens: 2 3 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758631934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758655566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758675998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758695358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758714951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758733432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:8  gens: 3 3 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758752595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:8  flush: 1 1 1 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758772582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758792326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:8  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758811758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758831632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758851398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.758870596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
test mend::test::reconcile_gen_b ... ok
{"msg":"generation number mismatch 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759362542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759393989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:0  gens: 1 1 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759414196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759434027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759453245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759472432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759492458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759510976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:1  gens: 2 1 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75953628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759557621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759576824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759596128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759615652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759633942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:2  gens: 3 1 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759653437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:2  flush: 1 1 1 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759673491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759693188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:2  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759712615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75973251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759752188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759771263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759790767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759809118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:3  gens: 1 2 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759828198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759847815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759866839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759885868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.75990527Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759927976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:4  gens: 2 2 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759948226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759968016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.759987345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76000659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760026375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760044697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:5  gens: 3 2 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760063765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:5  flush: 1 1 1 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760083553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760103236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:5  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760122528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760142577Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760162116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760181233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760200746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76021897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:6  gens: 1 3 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760238381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:6  flush: 1 1 1 scs: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760258238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760277873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:6  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760297158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76032121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760341911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760361312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760381124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760399515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:7  gens: 2 3 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760418711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:7  flush: 1 1 1 scs: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760438707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760458497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:7  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760477897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760497849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760517523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.760536778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
test mend::test::reconcile_gen_c ... ok
test mend::test::reconcile_gen_length_bad - should panic ... ok
{"msg":"generation number mismatch 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.761485359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.761516824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:0  gens: 9 8 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76153714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.761557209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.761576583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.761595891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
test mend::test::reconcile_generation_mismatch_c0 ... ok
{"msg":"generation number mismatch 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762035466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762074562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:0  gens: 9 8 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762096836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:0  flush: 2 2 2 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762116981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762136998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:0  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762156416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762176464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762196305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76221554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
my ef is: ExtentFix { source: ClientId(0), dest: [ClientId(1)] }
test mend::test::reconcile_generation_mismatch_c1 ... ok
{"msg":"generation number mismatch 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762666951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762698652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:0  gens: 7 8 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762719047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:0  flush: 2 2 2 scs: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762739247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762759272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:0  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762779122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76279936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 1 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762819272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762838582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762858584Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762877145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:1  gens: 8 9 10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762902155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 2 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762923317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762942749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762962202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"generation number mismatch 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.762981924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.763000413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:3  gens: 5 4 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76301999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.763039838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.763059166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.763078363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
test mend::test::reconcile_generation_mismatch_c2 ... ok
test mend::test::reconcile_length_mismatch - should panic ... ok
{"msg":"Extent 0 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.763974349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764005628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  gens: 9 9 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764067077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:0  flush: 1 2 3 scs: [ClientId(0), ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764088573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(1)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764108528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764128457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764147827Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764167135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 1 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764187223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764205721Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  gens: 7 8 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764231057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:1  flush: 1 1 2 scs: [ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764252529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76427227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764292467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764311807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764331266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 2 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764351193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764369768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:2  gens: 7 9 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764389471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 1 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764409308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764428512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 1, add dest 2 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764447816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"Extent 3 has flush number mismatch","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764467686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764486047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"extent:3  gens: 7 8 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764505484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"find dest for source 2 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764525339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.764544695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.7645641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"flush_mismatch"}
test mend::test::reconcile_multiple_source ... ok
test mend::test::reconcile_one ... ok
{"msg":"Extents 0 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765418816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765450762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:0  gens: 9 9 9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765476702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:0  flush: 2 2 3 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765498403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765517919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"find dest for source 2 for extent at index 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765537754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 2, add dest 0 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765557138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 2, add dest 1 flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765576225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"Extents 2 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765596033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765614764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:2  gens: 7 7 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765634034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"find dest for source 2 for extent at index 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765653604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 2, add dest 0 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765672711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 2, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765691798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"Extents 3 dirty","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765711247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76572971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:3  gens: 7 7 7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765749129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:3  flush: 1 1 1 scs: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765769434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"max_flush now has: [ClientId(0), ClientId(1), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765789597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"extent:3  dirty: true true true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765809039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"find dest for source 0 for extent at index 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765828849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 0, add dest 1 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765848237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"source 0, add dest 2 source flush","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765867673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"dirty"}
{"msg":"generation number mismatch 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765891371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"First source client ID for extent 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76591089Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:1  gens: 8 7 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765930598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:1  flush: 1 1 1 scs: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.765950615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"max_flush now has: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.76597072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"extent:1  dirty: false false false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.7659903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"No maxes found, left with: [ClientId(0), ClientId(2)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.766011081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"find dest for source 0 for extent at index 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.766031402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
{"msg":"source 0, add dest 1 gen","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.766050752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"mrl":"gen_mismatch"}
test mend::test::reconcile_one_of_each ... ok
test mend::test::reconcile_to_repair ... ok
{"msg":"Crucible stats registered with UUID: 3fdb42c8-a904-4e37-9001-6acceac3fb6c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.767026684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 3fdb42c8-a904-4e37-9001-6acceac3fb6c has session id: ffdd31cb-e7a1-4a7e-81ba-a8273c006d89","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.767060843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"3fdb42c8-a904-4e37-9001-6acceac3fb6c is now active with session: 6247eff3-a4fd-4994-9758-1b20f41c8bd7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.767082302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Decryption failed even though integrity hash matched!","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.767780195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Decryption failed with correct hash","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.767807379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error DecryptionError on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.76783853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::bad_decryption_means_panic ... ok
{"msg":"No match for integrity hash","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.768751878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Expected: 0x2710 != Computed: 0x5ce12454c95082b7","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.768789896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error HashMismatch on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.768822604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::bad_hash_on_encrypted_read_panic ... ok
{"msg":"Crucible stats registered with UUID: 4316b2ed-72a6-40cb-983c-23141001e682","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.769391235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 4316b2ed-72a6-40cb-983c-23141001e682 has session id: 7b325238-1b67-4ebf-934d-f99ead858efa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.769424791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"4316b2ed-72a6-40cb-983c-23141001e682 is now active with session: a45ed002-b2b6-4cb3-bb2d-fc5a888bd623","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.769445786Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No match computed hash:0x78fc2d7d9eaf9bbf","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.769464539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"No match          hash:0x2710","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.769484335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Data from hash:","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.769503472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.769522325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.769541201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.769560032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[3]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.76957858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[4]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.769597356Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[5]:1","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.769615943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error HashMismatch on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.769646062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::bad_read_hash_means_panic ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770234525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: d3548f3c-ca67-4037-ac1f-f010a1fdcf1c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770274168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 6a72f79d-a3f9-43d9-8241-0377b7bbf6e6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770296726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770316094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770407465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 00000000-0000-0000-0000-000000000000 (6a72f79d-a3f9-43d9-8241-0377b7bbf6e6) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770435536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77045498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770473589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 00000000-0000-0000-0000-000000000000 (6a72f79d-a3f9-43d9-8241-0377b7bbf6e6) Deactivated Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770494565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770513047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77053141Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770549492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770567685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770585754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770603953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770622026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 00000000-0000-0000-0000-000000000000 (6a72f79d-a3f9-43d9-8241-0377b7bbf6e6) Deactivated Active Deactivated ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77064279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770661233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770681288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770701028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770726095Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770745761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770764092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77078212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.770800171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"All DS in the proper state! -> INIT","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77081839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::deactivate_after_work_completed_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771459035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 703b59ba-34c1-4615-a19f-cb18d21c8d79","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771492659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: f41142b9-96b6-4b78-b743-60791b147d5c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771513536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771532574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771623337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 00000000-0000-0000-0000-000000000000 (f41142b9-96b6-4b78-b743-60791b147d5c) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771651199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77167021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771688148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 00000000-0000-0000-0000-000000000000 (f41142b9-96b6-4b78-b743-60791b147d5c) Deactivated Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771708722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771727092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771745398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771763462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771781573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771805819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Deactivated NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771825482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] check deactivate YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771843609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 00000000-0000-0000-0000-000000000000 (f41142b9-96b6-4b78-b743-60791b147d5c) Deactivated Active Deactivated ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771864601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771883076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771903189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77192293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 00000000-0000-0000-0000-000000000000 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771942751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771960936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.771979173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.7719975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.772016033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"All DS in the proper state! -> INIT","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.772034259Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::deactivate_after_work_completed_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: 59ffeb50-bbfb-468f-9838-3e246cb7bda2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77261573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 59ffeb50-bbfb-468f-9838-3e246cb7bda2 has session id: 1a8f88ad-9dba-4528-a293-b04c5c6577b0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77264939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"59ffeb50-bbfb-468f-9838-3e246cb7bda2 is now active with session: b07b796c-5eb3-4144-9382-1c890b42ec03","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.772670214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::deactivate_ds_not_when_active ... ok
{"msg":"Crucible stats registered with UUID: 2d69d5fd-8689-43e8-959d-eb511f22b975","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.773234251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 2d69d5fd-8689-43e8-959d-eb511f22b975 has session id: becd7e65-ee77-4283-b44b-1dba6911cf62","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.773267397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::deactivate_ds_not_when_initializing ... ok
{"msg":"Crucible stats registered with UUID: c9a38720-6349-48b1-9c95-f354263b69a6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.773826943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible c9a38720-6349-48b1-9c95-f354263b69a6 has session id: 7160363e-a509-43fd-8141-6136100bdc5d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.773865512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"c9a38720-6349-48b1-9c95-f354263b69a6 is now active with session: 5e0e2aa8-bd7e-461a-9cd7-dfaae181db17","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.773888091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"c9a38720-6349-48b1-9c95-f354263b69a6 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.773907188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::deactivate_not_when_active ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.774500816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: fe363f11-515b-4fe1-aaa4-b9af95aefa5f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.774533907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 2c4c5856-415d-4e76-8f68-945ddb52c496","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.774554771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77457359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.774627299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.774651251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77467043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.774688865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.774707219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.774725567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.774743889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::deactivate_not_without_flush_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.775297193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 8a054696-825c-412b-9fb8-8a34ca03e973","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.775330515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: de2da2cc-9019-4ae0-bcd5-7491dd6434b3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.775351331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.775370284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.775410399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77543189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] deactivate job 1001 not New flush, NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.775450869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.775468964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.775486914Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.775505084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition Active NO","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.775523048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::deactivate_not_without_flush_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: ff83e226-dd3f-4e44-be84-d52efdd6fcf2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776079794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ff83e226-dd3f-4e44-be84-d52efdd6fcf2 has session id: da861e5a-8204-44d7-98bc-bf94c06e4dc2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.7761131Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"ff83e226-dd3f-4e44-be84-d52efdd6fcf2 is now active with session: 3352050a-3e4c-4702-b20b-ff40606866a8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776134199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"ff83e226-dd3f-4e44-be84-d52efdd6fcf2 set deactivating.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776153291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] deactivate, no work so YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776171547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ff83e226-dd3f-4e44-be84-d52efdd6fcf2 (3352050a-3e4c-4702-b20b-ff40606866a8) Active Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776192332Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776211232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] deactivate, no work so YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776229733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ff83e226-dd3f-4e44-be84-d52efdd6fcf2 (3352050a-3e4c-4702-b20b-ff40606866a8) Deactivated Active Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77625057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776269165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] deactivate, no work so YES","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776287277Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ff83e226-dd3f-4e44-be84-d52efdd6fcf2 (3352050a-3e4c-4702-b20b-ff40606866a8) Deactivated Deactivated Active ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776308218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776332422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ff83e226-dd3f-4e44-be84-d52efdd6fcf2 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776353723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ff83e226-dd3f-4e44-be84-d52efdd6fcf2 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776373862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ff83e226-dd3f-4e44-be84-d52efdd6fcf2 Gone missing, transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776393686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate transition checking...","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776411945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776430182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776448526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"deactivate_transition New Maybe ","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776466672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"All DS in the proper state! -> INIT","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.776484917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::deactivate_when_empty ... ok
{"msg":"Crucible stats registered with UUID: 0dc9bb7c-f00c-46b4-b449-b29dea3c6f4b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.777045452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 0dc9bb7c-f00c-46b4-b449-b29dea3c6f4b has session id: 3d1e08b5-373b-41f1-8d6f-ec024985610c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.77707908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 0dc9bb7c-f00c-46b4-b449-b29dea3c6f4b (c37f3bec-904d-4191-822c-88c2c81b84df) New New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.777100908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_bad_transition_wq - should panic ... ok
{"msg":"Crucible stats registered with UUID: ead6ac30-3b89-489b-bdff-eba60a2fafb4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.777864401Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ead6ac30-3b89-489b-bdff-eba60a2fafb4 has session id: d7727c15-3078-4863-9753-dfd616730ec7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.777897468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ead6ac30-3b89-489b-bdff-eba60a2fafb4 (1dcc2031-1f3f-4663-b1df-92a89ad8f278) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.777919148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.777938006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ead6ac30-3b89-489b-bdff-eba60a2fafb4 (1dcc2031-1f3f-4663-b1df-92a89ad8f278) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.777958797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.777977383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ead6ac30-3b89-489b-bdff-eba60a2fafb4 (1dcc2031-1f3f-4663-b1df-92a89ad8f278) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778003886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778023825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ead6ac30-3b89-489b-bdff-eba60a2fafb4 (1dcc2031-1f3f-4663-b1df-92a89ad8f278) Active New New ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778044369Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778062764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_active_faulted ... ok
{"msg":"Crucible stats registered with UUID: ce47081e-609d-4bd9-96e0-3e9627a225cb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778625546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ce47081e-609d-4bd9-96e0-3e9627a225cb has session id: 84acc2f5-2a0a-48f4-a13b-902ed5bcbfe7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778658813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ce47081e-609d-4bd9-96e0-3e9627a225cb (832eddc2-7471-4a10-b7e5-35c0c66d0269) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778680493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778699142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ce47081e-609d-4bd9-96e0-3e9627a225cb (832eddc2-7471-4a10-b7e5-35c0c66d0269) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778719704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778738184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ce47081e-609d-4bd9-96e0-3e9627a225cb (832eddc2-7471-4a10-b7e5-35c0c66d0269) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778758512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778776912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ce47081e-609d-4bd9-96e0-3e9627a225cb (832eddc2-7471-4a10-b7e5-35c0c66d0269) Active New New ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778797346Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.778815826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_active_to_faulted ... ok
{"msg":"Crucible stats registered with UUID: d2b7cfe6-1c6e-4500-9d87-c67ccf233a2d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.779387403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible d2b7cfe6-1c6e-4500-9d87-c67ccf233a2d has session id: 473f5a84-3310-446b-8b96-2409bce79bc6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.779420084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] d2b7cfe6-1c6e-4500-9d87-c67ccf233a2d (23118f30-e980-4710-8f76-049fc3c3b60e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.779441807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.779466163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] d2b7cfe6-1c6e-4500-9d87-c67ccf233a2d (23118f30-e980-4710-8f76-049fc3c3b60e) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.779488213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.779506937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] d2b7cfe6-1c6e-4500-9d87-c67ccf233a2d (23118f30-e980-4710-8f76-049fc3c3b60e) WaitQuorum New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.779527735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_backwards - should panic ... ok
{"msg":"Crucible stats registered with UUID: 36f25b90-c77d-4d14-88f7-e35a97e003ec","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780135617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 36f25b90-c77d-4d14-88f7-e35a97e003ec has session id: dc40eb20-2983-4df1-82c4-e5685c017c3d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780168869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 36f25b90-c77d-4d14-88f7-e35a97e003ec (efe31aa6-7ba2-4b21-8776-d62aecd34153) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780190378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780209065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 36f25b90-c77d-4d14-88f7-e35a97e003ec (efe31aa6-7ba2-4b21-8776-d62aecd34153) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780229867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780248339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 36f25b90-c77d-4d14-88f7-e35a97e003ec (efe31aa6-7ba2-4b21-8776-d62aecd34153) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780268553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780286832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 36f25b90-c77d-4d14-88f7-e35a97e003ec (efe31aa6-7ba2-4b21-8776-d62aecd34153) Active New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780307101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_bad_active - should panic ... ok
{"msg":"Crucible stats registered with UUID: 7771bb8d-d0f4-46b1-a44d-c23c12ccdd11","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780907661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 7771bb8d-d0f4-46b1-a44d-c23c12ccdd11 has session id: 143c0113-42bf-445f-be5c-16b5d0122915","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780940337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7771bb8d-d0f4-46b1-a44d-c23c12ccdd11 (cc35b5ef-fbcb-41cd-a045-59af90473172) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780962268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.780986693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7771bb8d-d0f4-46b1-a44d-c23c12ccdd11 (cc35b5ef-fbcb-41cd-a045-59af90473172) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.781008925Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.781027722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7771bb8d-d0f4-46b1-a44d-c23c12ccdd11 (cc35b5ef-fbcb-41cd-a045-59af90473172) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.781048272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.781066654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7771bb8d-d0f4-46b1-a44d-c23c12ccdd11 (cc35b5ef-fbcb-41cd-a045-59af90473172) Active New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.781087083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.781105484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7771bb8d-d0f4-46b1-a44d-c23c12ccdd11 (cc35b5ef-fbcb-41cd-a045-59af90473172) Offline New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.781125853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_bad_offline - should panic ... ok
{"msg":"Crucible stats registered with UUID: 57c8db34-e48b-4a7d-83f3-8a88fa31a197","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.781728339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 57c8db34-e48b-4a7d-83f3-8a88fa31a197 has session id: 9a11e2c8-b8a9-48a3-9432-ed0899849ed4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.781761472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 57c8db34-e48b-4a7d-83f3-8a88fa31a197 (a7c8ef31-2f13-4be9-afc2-5125fa5aa8f3) New New New ds_transition to Replay","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.78178306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_bad_replay - should panic ... ok
{"msg":"Crucible stats registered with UUID: f7f80cb8-0fc7-47d1-b713-24a9a37d79c7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782391694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible f7f80cb8-0fc7-47d1-b713-24a9a37d79c7 has session id: 3ef0b24e-68a7-442d-ab20-dfb0ad2f2356","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782424583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f7f80cb8-0fc7-47d1-b713-24a9a37d79c7 (31b01c30-322a-49cd-af40-24c12e9ce431) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782446085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782464874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f7f80cb8-0fc7-47d1-b713-24a9a37d79c7 (31b01c30-322a-49cd-af40-24c12e9ce431) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782485413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782503844Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f7f80cb8-0fc7-47d1-b713-24a9a37d79c7 (31b01c30-322a-49cd-af40-24c12e9ce431) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782530204Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782550691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"f7f80cb8-0fc7-47d1-b713-24a9a37d79c7 is now active with session: 31b01c30-322a-49cd-af40-24c12e9ce431","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782571207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f7f80cb8-0fc7-47d1-b713-24a9a37d79c7 (31b01c30-322a-49cd-af40-24c12e9ce431) Active New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782591898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782610388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f7f80cb8-0fc7-47d1-b713-24a9a37d79c7 (31b01c30-322a-49cd-af40-24c12e9ce431) Deactivated New New ds_transition to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782630777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Deactivated to New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.782649241Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_deactivate_new ... ok
{"msg":"Crucible stats registered with UUID: 86e75c64-dac3-4542-a004-9168c4b81811","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.78320336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 86e75c64-dac3-4542-a004-9168c4b81811 has session id: 91a1f387-1762-4a79-a507-3646b8ac025b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.783237043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 86e75c64-dac3-4542-a004-9168c4b81811 (3ad53d45-4130-4e32-8e58-d7dad4efdd80) New New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.783258992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_deactivate_not_new - should panic ... ok
{"msg":"Crucible stats registered with UUID: 6f79ea9c-ea43-40f6-b86a-7c378a6b456e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.78387338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 6f79ea9c-ea43-40f6-b86a-7c378a6b456e has session id: be30719f-ba18-48fa-842d-91f3849e6905","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.78390666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6f79ea9c-ea43-40f6-b86a-7c378a6b456e (6959e13a-880e-4f13-914a-db664a6b3a0a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.783928121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.783946978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6f79ea9c-ea43-40f6-b86a-7c378a6b456e (6959e13a-880e-4f13-914a-db664a6b3a0a) WaitActive New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.783967717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_deactivate_not_wa - should panic ... ok
{"msg":"Crucible stats registered with UUID: 44c75964-0e56-4296-94f8-97ceb80595ec","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.784566066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 44c75964-0e56-4296-94f8-97ceb80595ec has session id: f3323cf3-fddd-45ec-aba7-f6577a8d0005","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.784605265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 44c75964-0e56-4296-94f8-97ceb80595ec (faa4d97e-b60a-4cc5-b158-d6ae6c9a634b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.784628282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.784647156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 44c75964-0e56-4296-94f8-97ceb80595ec (faa4d97e-b60a-4cc5-b158-d6ae6c9a634b) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.784667885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.784686361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 44c75964-0e56-4296-94f8-97ceb80595ec (faa4d97e-b60a-4cc5-b158-d6ae6c9a634b) WaitQuorum New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.784706946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_deactivate_not_wq - should panic ... ok
{"msg":"Crucible stats registered with UUID: 59cafd00-d67f-4bd7-af30-6e9787cb43f7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.785322613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 59cafd00-d67f-4bd7-af30-6e9787cb43f7 has session id: 04121b65-e7e7-4091-ba99-7696e7e4f0c2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.785356441Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 59cafd00-d67f-4bd7-af30-6e9787cb43f7 (1e7f6f83-82c5-49a5-9834-77da5dc3e264) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.78537802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.785397058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 59cafd00-d67f-4bd7-af30-6e9787cb43f7 (1e7f6f83-82c5-49a5-9834-77da5dc3e264) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.785417922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.785436413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 59cafd00-d67f-4bd7-af30-6e9787cb43f7 (1e7f6f83-82c5-49a5-9834-77da5dc3e264) WaitQuorum New New ds_transition to Deactivated","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.785457061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_disconnect_no_active - should panic ... ok
{"msg":"Crucible stats registered with UUID: bfd19920-0b05-47a5-828b-205cd56f5687","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786064807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible bfd19920-0b05-47a5-828b-205cd56f5687 has session id: e96c1960-330a-4a85-929e-b1de77ccfa65","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786098198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] bfd19920-0b05-47a5-828b-205cd56f5687 (f3b84223-32dd-4976-ae7b-aa4c14635261) New New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786119476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_no_new_to_offline - should panic ... ok
{"msg":"Crucible stats registered with UUID: 7abd282f-4431-4a20-95df-54ccb48a0671","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786738339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 7abd282f-4431-4a20-95df-54ccb48a0671 has session id: 686b946e-3de7-4d48-b606-577081f18781","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786771136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7abd282f-4431-4a20-95df-54ccb48a0671 (07b84a96-4838-43f4-bc72-cd6ff41826b6) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786792594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786811369Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7abd282f-4431-4a20-95df-54ccb48a0671 (07b84a96-4838-43f4-bc72-cd6ff41826b6) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786831832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786850213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 7abd282f-4431-4a20-95df-54ccb48a0671 (07b84a96-4838-43f4-bc72-cd6ff41826b6) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786870632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.786888996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_normal ... ok
{"msg":"Crucible stats registered with UUID: 2752c185-b27a-47b3-80bf-ed0f9e6e8a44","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787468523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 2752c185-b27a-47b3-80bf-ed0f9e6e8a44 has session id: 8466f959-c80c-4506-a44e-83ebc36b1ddc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787501373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2752c185-b27a-47b3-80bf-ed0f9e6e8a44 (f5db4d2d-bf63-4549-8328-92ea47aff629) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787522963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787541569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2752c185-b27a-47b3-80bf-ed0f9e6e8a44 (f5db4d2d-bf63-4549-8328-92ea47aff629) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787562275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787580911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2752c185-b27a-47b3-80bf-ed0f9e6e8a44 (f5db4d2d-bf63-4549-8328-92ea47aff629) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787601252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787619647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2752c185-b27a-47b3-80bf-ed0f9e6e8a44 (f5db4d2d-bf63-4549-8328-92ea47aff629) Active New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787640204Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787664442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2752c185-b27a-47b3-80bf-ed0f9e6e8a44 (f5db4d2d-bf63-4549-8328-92ea47aff629) Offline New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.787686328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_offline_no_active - should panic ... ok
{"msg":"Crucible stats registered with UUID: e8eaa90b-d486-4bbb-8e96-dccd88d40eb6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788289676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible e8eaa90b-d486-4bbb-8e96-dccd88d40eb6 has session id: b7b7d676-a503-4725-abc6-176a68860526","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788322779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e8eaa90b-d486-4bbb-8e96-dccd88d40eb6 (23b59aff-5c96-4189-9cb9-518bc8aaa60a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788344618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788363656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e8eaa90b-d486-4bbb-8e96-dccd88d40eb6 (23b59aff-5c96-4189-9cb9-518bc8aaa60a) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788384427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788403175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"e8eaa90b-d486-4bbb-8e96-dccd88d40eb6 is now active with session: 23b59aff-5c96-4189-9cb9-518bc8aaa60a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788423218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e8eaa90b-d486-4bbb-8e96-dccd88d40eb6 (23b59aff-5c96-4189-9cb9-518bc8aaa60a) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788443896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.78846232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e8eaa90b-d486-4bbb-8e96-dccd88d40eb6 (23b59aff-5c96-4189-9cb9-518bc8aaa60a) Active New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788483034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788501524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e8eaa90b-d486-4bbb-8e96-dccd88d40eb6 (23b59aff-5c96-4189-9cb9-518bc8aaa60a) Offline New New ds_transition to Replay","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788521923Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Offline to Replay","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.788540261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_replay ... ok
{"msg":"Crucible stats registered with UUID: 6e5e04a4-4f4b-4d45-b79f-e4d4b7899dc2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.78909506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 6e5e04a4-4f4b-4d45-b79f-e4d4b7899dc2 has session id: 501a0f67-5e4d-4a28-bad0-20d91ec74a06","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789128773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6e5e04a4-4f4b-4d45-b79f-e4d4b7899dc2 (dace7cdf-410a-447a-a581-1976a197f1a8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789156243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789176631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6e5e04a4-4f4b-4d45-b79f-e4d4b7899dc2 (dace7cdf-410a-447a-a581-1976a197f1a8) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789197447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789215882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6e5e04a4-4f4b-4d45-b79f-e4d4b7899dc2 (dace7cdf-410a-447a-a581-1976a197f1a8) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789236438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789254755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 6e5e04a4-4f4b-4d45-b79f-e4d4b7899dc2 (dace7cdf-410a-447a-a581-1976a197f1a8) Active New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789275202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_same_active - should panic ... ok
{"msg":"Crucible stats registered with UUID: 3e38a6a4-5ad1-4517-9c5a-ee2835e33962","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789873671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 3e38a6a4-5ad1-4517-9c5a-ee2835e33962 has session id: 31fa5b18-879c-403d-a5f4-6e235696282e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789906628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3e38a6a4-5ad1-4517-9c5a-ee2835e33962 (b372a068-e3b2-4f5c-a122-c7947a92b044) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789928218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789947135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3e38a6a4-5ad1-4517-9c5a-ee2835e33962 (b372a068-e3b2-4f5c-a122-c7947a92b044) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789967939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.789986584Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3e38a6a4-5ad1-4517-9c5a-ee2835e33962 (b372a068-e3b2-4f5c-a122-c7947a92b044) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.7900072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.790025885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3e38a6a4-5ad1-4517-9c5a-ee2835e33962 (b372a068-e3b2-4f5c-a122-c7947a92b044) Active New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.790046468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.790064709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3e38a6a4-5ad1-4517-9c5a-ee2835e33962 (b372a068-e3b2-4f5c-a122-c7947a92b044) Offline New New ds_transition to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.790090668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_same_offline - should panic ... ok
{"msg":"Crucible stats registered with UUID: 38a9c65e-bedc-4ca2-b716-2583bb874a36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.790699407Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 38a9c65e-bedc-4ca2-b716-2583bb874a36 has session id: b134edeb-e702-4f4c-b1f8-c009dbb4c307","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.790732817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 38a9c65e-bedc-4ca2-b716-2583bb874a36 (c2236e94-8c87-4dfa-8447-5bbef3843952) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.790754825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.790773328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 38a9c65e-bedc-4ca2-b716-2583bb874a36 (c2236e94-8c87-4dfa-8447-5bbef3843952) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.790794071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_same_wa - should panic ... ok
{"msg":"Crucible stats registered with UUID: 2e0b2db2-d735-47ea-9950-19328964ab66","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.791396617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 2e0b2db2-d735-47ea-9950-19328964ab66 has session id: c4172829-b2a6-423c-b24c-512ca4f30acf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.791429701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2e0b2db2-d735-47ea-9950-19328964ab66 (daf5c384-9e01-4130-a91f-ebf2171835bb) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.791451138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.791469996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2e0b2db2-d735-47ea-9950-19328964ab66 (daf5c384-9e01-4130-a91f-ebf2171835bb) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.791490717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.791509232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2e0b2db2-d735-47ea-9950-19328964ab66 (daf5c384-9e01-4130-a91f-ebf2171835bb) WaitQuorum New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.791529687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::downstairs_transition_same_wq - should panic ... ok
{"msg":"Crucible stats registered with UUID: 43ece84c-fadd-47ba-bae7-1f003b9b46ee","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792137463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 43ece84c-fadd-47ba-bae7-1f003b9b46ee has session id: 78eb4c19-2f8c-4c5e-9d45-53e542429b06","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792170412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 43ece84c-fadd-47ba-bae7-1f003b9b46ee (4863f532-0cf8-4602-bd3c-3435618d09b9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792191882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792216333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 43ece84c-fadd-47ba-bae7-1f003b9b46ee (4863f532-0cf8-4602-bd3c-3435618d09b9) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792238386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792256929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 43ece84c-fadd-47ba-bae7-1f003b9b46ee (4863f532-0cf8-4602-bd3c-3435618d09b9) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792277441Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79229595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 43ece84c-fadd-47ba-bae7-1f003b9b46ee (4863f532-0cf8-4602-bd3c-3435618d09b9) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792316467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792334626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 43ece84c-fadd-47ba-bae7-1f003b9b46ee (4863f532-0cf8-4602-bd3c-3435618d09b9) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792355411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792373859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 43ece84c-fadd-47ba-bae7-1f003b9b46ee (4863f532-0cf8-4602-bd3c-3435618d09b9) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792394328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792412593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 43ece84c-fadd-47ba-bae7-1f003b9b46ee (4863f532-0cf8-4602-bd3c-3435618d09b9) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79243303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79245118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 43ece84c-fadd-47ba-bae7-1f003b9b46ee (4863f532-0cf8-4602-bd3c-3435618d09b9) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792471756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79249003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 43ece84c-fadd-47ba-bae7-1f003b9b46ee (4863f532-0cf8-4602-bd3c-3435618d09b9) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792510639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792528949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"43ece84c-fadd-47ba-bae7-1f003b9b46ee is now active with session: 4863f532-0cf8-4602-bd3c-3435618d09b9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792552843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 43ece84c-fadd-47ba-bae7-1f003b9b46ee (4863f532-0cf8-4602-bd3c-3435618d09b9) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792574153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.792592435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::faulted_downstairs_skips_but_still_does_work ... ok
{"msg":"Crucible stats registered with UUID: a8cb0ee1-4130-4193-8ea1-a0aa4ed74780","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793163381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 has session id: 1913e096-ef3f-4674-a90d-74a2a7f0d5dd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793196486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 (9ccf2467-2665-45a9-86f3-6dee7a59fac5) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793218312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793236972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 (9ccf2467-2665-45a9-86f3-6dee7a59fac5) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793257732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793275968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 (9ccf2467-2665-45a9-86f3-6dee7a59fac5) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793296281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793314408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 (9ccf2467-2665-45a9-86f3-6dee7a59fac5) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793334666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793352696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 (9ccf2467-2665-45a9-86f3-6dee7a59fac5) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793373353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793391716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 (9ccf2467-2665-45a9-86f3-6dee7a59fac5) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793411976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793429988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 (9ccf2467-2665-45a9-86f3-6dee7a59fac5) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793456305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793476335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 (9ccf2467-2665-45a9-86f3-6dee7a59fac5) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793497136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793515718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 (9ccf2467-2665-45a9-86f3-6dee7a59fac5) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793536098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793554231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 is now active with session: 9ccf2467-2665-45a9-86f3-6dee7a59fac5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793573898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] a8cb0ee1-4130-4193-8ea1-a0aa4ed74780 (9ccf2467-2665-45a9-86f3-6dee7a59fac5) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793594241Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.793612284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::faulted_downstairs_skips_work ... ok
{"msg":"Crucible stats registered with UUID: 3dd83520-4b07-4cfa-9bd0-add451f7ea4c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79417941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 3dd83520-4b07-4cfa-9bd0-add451f7ea4c has session id: 24c704a8-cffe-4af2-8011-77aa3d444870","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794212499Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"3dd83520-4b07-4cfa-9bd0-add451f7ea4c is now active with session: 329df8b1-93de-44e0-b9b2-22eb23edd7b6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794233639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794254173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794272845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794293265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794311643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794332088Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794356706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794378639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794397166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794417886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79443641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794456905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794475117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794495706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794514046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794534753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794553187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794573935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794592399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794612971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794631274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 3dd83520-4b07-4cfa-9bd0-add451f7ea4c (329df8b1-93de-44e0-b9b2-22eb23edd7b6) Active Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794651885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.794670058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::flush_io_double_skip ... ok
{"msg":"Crucible stats registered with UUID: 64fac11f-725d-4d02-bf9e-048fc18f4c91","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795232624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 64fac11f-725d-4d02-bf9e-048fc18f4c91 has session id: e5af5ee5-91a0-49c0-ae2b-b29606b8f641","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795270635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"64fac11f-725d-4d02-bf9e-048fc18f4c91 is now active with session: 460cda23-2385-4edd-840e-6c135a560abf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795293226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 64fac11f-725d-4d02-bf9e-048fc18f4c91 (460cda23-2385-4edd-840e-6c135a560abf) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795313891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795332613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 64fac11f-725d-4d02-bf9e-048fc18f4c91 (460cda23-2385-4edd-840e-6c135a560abf) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795353117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795371603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 64fac11f-725d-4d02-bf9e-048fc18f4c91 (460cda23-2385-4edd-840e-6c135a560abf) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795392025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795410186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 64fac11f-725d-4d02-bf9e-048fc18f4c91 (460cda23-2385-4edd-840e-6c135a560abf) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795430541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795448664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 64fac11f-725d-4d02-bf9e-048fc18f4c91 (460cda23-2385-4edd-840e-6c135a560abf) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795469186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795487585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 64fac11f-725d-4d02-bf9e-048fc18f4c91 (460cda23-2385-4edd-840e-6c135a560abf) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79550811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795526605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 64fac11f-725d-4d02-bf9e-048fc18f4c91 (460cda23-2385-4edd-840e-6c135a560abf) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795546906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795565002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 64fac11f-725d-4d02-bf9e-048fc18f4c91 (460cda23-2385-4edd-840e-6c135a560abf) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795585521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795607914Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 64fac11f-725d-4d02-bf9e-048fc18f4c91 (460cda23-2385-4edd-840e-6c135a560abf) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795629427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79564793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 64fac11f-725d-4d02-bf9e-048fc18f4c91 (460cda23-2385-4edd-840e-6c135a560abf) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79566838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.795686679Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: Flush { dependencies: [], flush_number: 22, gen_number: 11, snapshot_details: None, extent_limit: None }, state: ClientData([Skipped, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.795715422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: Flush { dependencies: [], flush_number: 22, gen_number: 11, snapshot_details: None, extent_limit: None }, state: ClientData([Skipped, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.795745241Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::flush_io_fail_and_skip ... ok
{"msg":"Crucible stats registered with UUID: 90e4abfa-c634-471a-a9d8-50c07c1dff79","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796321375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 90e4abfa-c634-471a-a9d8-50c07c1dff79 has session id: 3fb10cc9-f589-4d77-b07a-c737e26169db","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796366322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"90e4abfa-c634-471a-a9d8-50c07c1dff79 is now active with session: e7c36b35-a726-4f02-a6b2-b774d20209eb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796389301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 90e4abfa-c634-471a-a9d8-50c07c1dff79 (e7c36b35-a726-4f02-a6b2-b774d20209eb) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79641002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79642867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 90e4abfa-c634-471a-a9d8-50c07c1dff79 (e7c36b35-a726-4f02-a6b2-b774d20209eb) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796449153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796467524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 90e4abfa-c634-471a-a9d8-50c07c1dff79 (e7c36b35-a726-4f02-a6b2-b774d20209eb) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796487834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796511781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 90e4abfa-c634-471a-a9d8-50c07c1dff79 (e7c36b35-a726-4f02-a6b2-b774d20209eb) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796533215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796551475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 90e4abfa-c634-471a-a9d8-50c07c1dff79 (e7c36b35-a726-4f02-a6b2-b774d20209eb) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796571994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796590172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 90e4abfa-c634-471a-a9d8-50c07c1dff79 (e7c36b35-a726-4f02-a6b2-b774d20209eb) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796610387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796628556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 90e4abfa-c634-471a-a9d8-50c07c1dff79 (e7c36b35-a726-4f02-a6b2-b774d20209eb) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796649129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796667055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 90e4abfa-c634-471a-a9d8-50c07c1dff79 (e7c36b35-a726-4f02-a6b2-b774d20209eb) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.79668744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796705651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 90e4abfa-c634-471a-a9d8-50c07c1dff79 (e7c36b35-a726-4f02-a6b2-b774d20209eb) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796725939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796743906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 90e4abfa-c634-471a-a9d8-50c07c1dff79 (e7c36b35-a726-4f02-a6b2-b774d20209eb) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796764187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.796782318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::flush_io_single_skip ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.797352969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 061eba66-4173-4774-97fd-aadd36dd85d5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.797386055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::not_right_block_size - should panic ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.798093424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 7fefaa67-fc26-4742-b884-35f4bb7d8abb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.798126627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::off_to_extent_bridge ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.800902284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 8248d1ce-196b-4603-b2f3-860a3a1dd925","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.800935509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::off_to_extent_length_almost_too_big ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.80159858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 9914ac4f-aa60-463d-8702-ed1f285919e5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.801632137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::off_to_extent_length_and_offset_almost_too_big ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.802223727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: c00d0775-ab92-4694-aebf-86174f45c437","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.802256985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::off_to_extent_length_and_offset_too_big - should panic ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.802874686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 595854ca-954b-4521-bebb-1bb6a0b4e9a8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.802907256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::off_to_extent_length_too_big - should panic ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.803521879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: f265562f-d096-464e-8514-7cb7b25a1eb7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.803554946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::off_to_extent_length_zero ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.804112672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 327f36c8-3c7d-4616-a13a-d2ffb802df07","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.804146161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::off_to_extent_one_block ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.805130471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 8d094510-cdd3-415e-b451-8328a5ee0443","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.805169753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::off_to_extent_two_blocks ... ok
{"msg":"Crucible stats registered with UUID: 0a5ceb5e-a244-4fe3-9861-83ccd3099205","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806177256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 0a5ceb5e-a244-4fe3-9861-83ccd3099205 has session id: 1b2a4902-676a-4e1a-87e3-cb0e7c3cbe08","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806210099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806232052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806250869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806271764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.80629047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806311134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806329551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806350081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806368381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806389238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806407644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806428143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806446519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806466925Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806485276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806511909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806531845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806552372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806570565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"0a5ceb5e-a244-4fe3-9861-83ccd3099205 is now active with session: a60075df-ec91-455d-8fe1-c145ba0f2a55","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806590315Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.806623395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.806658713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806679415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806698796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806720324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.80673871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.806772118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.806810247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806831562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806851181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 0a5ceb5e-a244-4fe3-9861-83ccd3099205 (a60075df-ec91-455d-8fe1-c145ba0f2a55) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.806873673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.80689232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::read_after_two_write_fail_is_alright ... ok
{"msg":"Crucible stats registered with UUID: 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807462276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 has session id: 6862594b-a5e0-427f-95ca-728133c7c3c4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807495615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 (5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807517409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807536156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 (5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807556839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807575282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 (5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.80759547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.80761376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 (5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807634013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807652045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 (5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807678668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807698226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 (5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807718713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807736898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 (5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.80775738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807775536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 (5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807795841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807814125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 (5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807834621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807852932Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 is now active with session: 5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807872571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.807905429Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.807939693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807964782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.807985405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 8bfbf34a-c7f4-4b7f-9fdd-7cec799871e8 (5b5e90bd-53e9-47e8-b26d-ccd08acbdeaa) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.808007936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.808026397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::read_after_write_fail_is_alright ... ok
{"msg":"Crucible stats registered with UUID: 0b89c204-7ea7-4b26-8e17-efa6d769eee7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.808609457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 0b89c204-7ea7-4b26-8e17-efa6d769eee7 has session id: 5d48fae0-e5b7-4078-a58c-af8e75c04681","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.808660532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.808685747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.808708528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.808733036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.808756638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.808779808Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::reconcile_leave_no_job_behind - should panic ... ok
{"msg":"Crucible stats registered with UUID: bbf965c8-2f32-4092-ad08-341946975de0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.809390366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible bbf965c8-2f32-4092-ad08-341946975de0 has session id: e0e46971-08b9-45db-b39e-ed0707a1cef0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.809423573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] bbf965c8-2f32-4092-ad08-341946975de0 (0be11dd1-6309-4977-a837-3113bf96dbf7) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.809445291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.809463956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] bbf965c8-2f32-4092-ad08-341946975de0 (0be11dd1-6309-4977-a837-3113bf96dbf7) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.80949034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.809510454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] bbf965c8-2f32-4092-ad08-341946975de0 (0be11dd1-6309-4977-a837-3113bf96dbf7) WaitQuorum New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.809531071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.809549403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] bbf965c8-2f32-4092-ad08-341946975de0 (0be11dd1-6309-4977-a837-3113bf96dbf7) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.809570117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.809588774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.809607135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::reconcile_not_ready ... ok
{"msg":"Crucible stats registered with UUID: 1096010d-5ed4-4780-9317-bfd7ffb0ff17","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.810179294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 1096010d-5ed4-4780-9317-bfd7ffb0ff17 has session id: 6252da48-dccd-4eeb-b3cb-f42ab9e3f698","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.810212383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Full repair list: {9: ExtentFix { source: ClientId(0), dest: [ClientId(1), ClientId(2)] }}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.810233253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 9, client_id: ClientId(0), flush_number: 22, gen_number: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 9 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 9, source_client_id: ClientId(0), source_repair_address: 127.0.0.1:801, dest_clients: [ClientId(1), ClientId(2)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 9 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.810274587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::reconcile_rc_to_message ... ok
{"msg":"Crucible stats registered with UUID: d30f6c89-428e-48b7-877d-8aa2c23da5a2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.810858831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible d30f6c89-428e-48b7-877d-8aa2c23da5a2 has session id: a97256fe-a31d-43d6-a5a4-e09fffabd5dc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.810891755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Full repair list: {5: ExtentFix { source: ClientId(2), dest: [ClientId(0), ClientId(1)] }}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.811021081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 5, client_id: ClientId(2), flush_number: 66, gen_number: 77 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 5 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 5, source_client_id: ClientId(2), source_repair_address: 127.0.0.1:803, dest_clients: [ClientId(0), ClientId(1)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 5 }, state: ClientData([New, New, New]) }]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.811072195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::reconcile_rc_to_message_two ... ok
{"msg":"Crucible stats registered with UUID: c065c25d-9b93-4776-b10a-88edf0f4668d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.811642585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible c065c25d-9b93-4776-b10a-88edf0f4668d has session id: b9421b5b-fd28-41ba-96dd-46815b9abce9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.811676271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.811699336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::reconcile_rep_done_too_soon - should panic ... ok
{"msg":"Crucible stats registered with UUID: 84fa6ec6-e5be-4e13-b154-a89c96163be0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.812324331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 84fa6ec6-e5be-4e13-b154-a89c96163be0 has session id: f7795422-1eb4-4bfa-b4c4-e4754e16a099","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.812357144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.81238013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.812402294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] rep_in_progress ignore submitted job ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.812426512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::reconcile_rep_in_progress_bad1 - should panic ... ok
{"msg":"Crucible stats registered with UUID: 2e39358b-3ac9-4955-ab6e-32a34bdd71d8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.813032231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 2e39358b-3ac9-4955-ab6e-32a34bdd71d8 has session id: 8f21b75e-3ea2-47e8-940b-816bc1d88370","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.813065562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::reconcile_rep_in_progress_none ... ok
{"msg":"Crucible stats registered with UUID: 1f19f65d-2351-4f27-b314-11785e0566df","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.813623867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 1f19f65d-2351-4f27-b314-11785e0566df has session id: 64740696-b9f7-489d-96fa-a766864012b5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.813662274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.813686958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::reconcile_repair_inprogress_not_done - should panic ... ok
{"msg":"Crucible stats registered with UUID: 0628d57e-31e0-4662-9521-c7b8bb6fef09","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.814303739Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 0628d57e-31e0-4662-9521-c7b8bb6fef09 has session id: 7f5cbd16-7b78-44fb-b4d9-571fe0dfbfee","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.814337373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.81435993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.814382396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.81440616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.814429556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.814451656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.814473273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.81449627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.814519641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::reconcile_repair_workflow_1 ... ok
{"msg":"Crucible stats registered with UUID: d839dc93-0b6c-4d12-8978-7d3523282044","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.815084901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible d839dc93-0b6c-4d12-8978-7d3523282044 has session id: baaaad4e-f367-461a-af99-145df3770fe2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.815122967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.815147319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.815169874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, Skipped, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.815194039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::reconcile_repair_workflow_2 ... ok
{"msg":"Crucible stats registered with UUID: 342ea606-82b9-4a22-9948-f23af258a77a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.815760302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 342ea606-82b9-4a22-9948-f23af258a77a has session id: 768f12da-4a5e-48b6-9603-65336a434a7a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.815793468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Mark 0 as FAILED REPAIR","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.815812805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Mark 2 as FAILED REPAIR","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.815832189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Clear out existing repair work queue","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.815851788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::reconcile_repair_workflow_not_repair ... ok
{"msg":"Crucible stats registered with UUID: 045bf5a0-f24b-4e5d-8eee-5cd365625bb9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.816420682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 045bf5a0-f24b-4e5d-8eee-5cd365625bb9 has session id: 5deed47f-0e87-489c-b858-8a7052767c4f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.816454016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.816476952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.816499503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.816523451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.816546994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Mark 0 as FAILED REPAIR","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.816571764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Mark 2 as FAILED REPAIR","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.816592322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Clear out existing repair work queue","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.816611858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::reconcile_repair_workflow_not_repair_later ... ok
{"msg":"Crucible stats registered with UUID: dbe048ac-a1fe-4b72-a0aa-f525ae7d3cd2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.817171556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible dbe048ac-a1fe-4b72-a0aa-f525ae7d3cd2 has session id: 9bacc61f-055d-4854-8801-b6b9cc396b23","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.817205286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.817228119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.817250294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.817274193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.817297386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::reconcile_repair_workflow_repair_later ... ok
{"msg":"Crucible stats registered with UUID: ad0ad087-5636-4dbb-9b1f-4e7162aaf473","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.817861569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ad0ad087-5636-4dbb-9b1f-4e7162aaf473 has session id: 4a62f8ec-8f60-49a6-971d-73bd97648a14","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.817894875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentClose { repair_id: ReconciliationId(0), extent_id: 1 }, state: ClientData([New, New, New]) }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.817917412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::reconcile_repair_workflow_too_soon - should panic ... ok
{"msg":"Crucible stats registered with UUID: 9b0d7727-40bf-4713-94eb-685aef15650d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.818537539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 9b0d7727-40bf-4713-94eb-685aef15650d has session id: 9ae2b501-1c6c-427a-8268-fe90a7916390","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.818570436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"9b0d7727-40bf-4713-94eb-685aef15650d is now active with session: ba963a29-d324-4d03-b68c-c282aea5ecfc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.818596976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[rc] leave job 1000 on the queue when removing 1001 WorkCounts { active: 2, error: 0, skipped: 0, done: 1 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.818622164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::retire_dont_retire_everything ... ok
test test::up_test::send_io_live_repair_read ... ok
test test::up_test::send_io_live_repair_unwritten_write ... ok
test test::up_test::send_io_live_repair_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.820034542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 400ccb50-91c9-4748-87ad-eed7797a135d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.820067858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 8026c04b-c42e-4254-90e4-dc5293998b77","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.820088608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_all_writes_depend_on_flushes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.820963128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 9c4aeccc-a39e-4f93-9224-ab443220b887","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.820996459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 4b13766d-1c35-44bd-97a3-3f7dfb293c0c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.821017661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_big_read_depends_on_little_writes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.821767272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 3e9855cf-470f-49ac-918b-e26876852912","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.821800442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 49984a05-6b64-4e70-a1f4-90987c78e84f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.821821429Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_big_write_depends_on_little_writes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.82256672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: c2a0fb8f-ef49-4f5a-9a05-8860414a6280","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.822599901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 2b55c596-21e4-4a7e-a578-388b3a9bcf1e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.822621041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_depend_on_acked_work ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.823328009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 55ee8f87-8d1d-4e94-8894-c052b9b692f5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.823366577Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 705a17e5-2505-4c68-a7df-f68f5a16a90c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.823389206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_flushes_depend_on_flushes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.824037225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: f56910f3-efd3-4de7-afaf-7d6accd13d87","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.824070147Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: f92ee978-1a92-44c5-8d7c-6007c8bb7bb5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.824091105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_flushes_depend_on_flushes_and_all_writes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.824962514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: d22fa24e-acd4-4706-aaaf-9ed18ffa17a2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.824995445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 1d0a3a03-deb8-4faa-94ab-faff91b11b40","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.825059888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_little_writes_depend_on_big_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.825767211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 01425d2c-36fb-4ea3-8623-74aa5d0faa5b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.825800105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 8d3d8f32-5bae-45e2-afa3-e7ea4d82fc31","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.825821122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_little_writes_depend_on_big_write_chain ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.826648684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: ba898aea-2870-4e77-8cda-df85c7b6ddbe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.826681905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: aaa4205d-523a-4668-a5a0-ebe56f00f290","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.826703117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_multi_extent_batman ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.827466363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 2f03f070-8e48-4ae1-ba83-3570ef59e3ca","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.82749892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 32fb8017-0f69-44f5-bd03-e7b3cc5263a6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.827525731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_multi_extent_there_and_back_again ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.828375766Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 98c73ec0-8fef-495f-a57b-b497944d25eb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.828409047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: cd9b78ef-6883-42e6-9dc8-4df3da57cea5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.828430114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_multi_extent_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.829171735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: fe3063a7-85d9-4920-925d-fe47b0464524","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.829204836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 0a8341e4-2e95-4877-abda-cd13672f2ec2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.829225861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_multiple_reads_depend_on_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.829906187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 20ea752f-11ff-45e7-9739-3337d54ea0f8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.8299395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 5fd49c0d-4371-4544-ae12-37b2a0440c29","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.829960854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_read_depends_on_flush ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.830645367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 48714655-f322-46b9-ac70-e785c3994869","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.830678331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 925aa39e-6824-4e72-bc51-9a85b7fa8392","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.830699223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_read_depends_on_write ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.831346818Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 88ebc32d-c873-4e0c-84f5-4d960c35a5bd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.831379844Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: cbc5cc40-5f88-41bd-b66e-218569c8df9c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.831406245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_read_no_depend_on_read ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.832044018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: d9610d11-d69b-4620-ac5f-9562c46c288b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.832077119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: ca97c4e0-1919-4d0b-aeb4-cf397f4dabe7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.832098094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_read_write_batman ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.832799807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: f8b0e600-168e-439a-a50e-8754fdcca0fe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.83283314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 8887bd21-6151-4472-9cb4-7a76ea21ade4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.832854038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_read_write_ladder_1 ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.833629455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 095a7024-45f2-4a3f-81ae-92c9a766278f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.833662345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 90d57ae5-1f02-4ffa-b6ed-2c94cb4e3e09","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.833683233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_read_write_ladder_2 ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.834410021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 82a703c3-f9a8-4153-a21b-e85cbb19ede6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.834443091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: f4e63527-5021-4e21-acf9-1e191852240e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.834464016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_read_write_ladder_3 ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.835216123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: b2d843e7-e553-4e4d-b4a7-c971a147dae1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.835248687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 9770c79d-b115-4452-b2b4-e6b6abc0fc27","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.835269729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_write_unwrittens_depend_on_read ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.835946126Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: fd3652d6-b8c8-4cc1-a493-9287e468bbd9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.835979253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 8518cf9a-fb55-4c27-bab2-dc9059d42c26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.836000278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_writes_depend_on_overlapping_writes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.836666858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 13d29be0-8f00-4a8e-9ba9-d45fd3756542","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.836699891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: ee73f036-d026-4e56-890e-7f7380d872e4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.836720926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_writes_depend_on_overlapping_writes_and_flushes ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.837403742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: a2ff5a1f-78b8-4ea3-ac4d-ec560dad768f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.837437289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: 1d090668-176a-496c-9a94-b5de202b9d61","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.837458052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_writes_depend_on_overlapping_writes_chain ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.8381479Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: ce8c253a-d0c3-40af-aaa3-8598e5065cd3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.838181006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: cb90bcd6-2d66-4b22-8d0b-a666140843f0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.838206881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_deps_writes_depend_on_read ... ok
test test::up_test::test_flush_does_not_consume_bw ... ok
test test::up_test::test_flush_does_not_consume_iops ... ok
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.851543264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.851608986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.851647351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.851689319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.851725288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.851760708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.851794561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.851827583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 36f6e1bc-5305-47e3-91bd-fe997b9223d6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852294013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 36f6e1bc-5305-47e3-91bd-fe997b9223d6 has session id: 3c815966-8e5b-497b-b01e-7b8c2cf11054","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852349652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852391942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852428779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.85246815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852506601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.85254886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.85258774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852629452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852667693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852692586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852711255Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852731648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852749937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852776914Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852796407Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852817266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852835688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852856328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852874527Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"36f6e1bc-5305-47e3-91bd-fe997b9223d6 is now active with session: 98b45c36-052b-4d2c-8209-f6db7df5fa0e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852894391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852914963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852933083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852953963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852972451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.852993119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.85301153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.85302985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.853048627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.853067049Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.853085706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_impossible_io ... ok
test test::up_test::test_iop_and_bw_limit ... ok
test test::up_test::test_iospan ... ok
test test::up_test::test_iospan_buffer_read_write ... ok
test test::up_test::test_no_iop_limit ... ok
{"msg":"Crucible stats registered with UUID: 00000000-0000-0000-0000-000000000000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.889227851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00000000-0000-0000-0000-000000000000 has session id: 35fe92bd-7d43-46d8-8864-a73306208500","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.889277718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00000000-0000-0000-0000-000000000000 is now active with session: ae6fec1e-970f-40b4-a78f-9ce644a4f5fa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.889298712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::test_read_flush_write_hash_mismatch ... ok
test test::up_test::test_set_bw_limit ... ok
test test::up_test::test_set_iop_limit ... ok
test test::up_test::test_upstairs_encryption_context_ok ... ok
test test::up_test::test_upstairs_encryption_context_wrong_nonce ... ok
test test::up_test::test_upstairs_encryption_context_wrong_tag ... ok
test test::up_test::test_upstairs_validate_encrypted_read_response ... ok
test test::up_test::test_upstairs_validate_encrypted_read_response_blank_block ... ok
test test::up_test::test_upstairs_validate_encrypted_read_response_multiple_contexts ... ok
test test::up_test::test_upstairs_validate_unencrypted_read_response ... ok
test test::up_test::test_upstairs_validate_unencrypted_read_response_blank_block ... ok
test test::up_test::test_upstairs_validate_unencrypted_read_response_multiple_contexts ... ok
test test::up_test::test_upstairs_validate_unencrypted_read_response_multiple_hashes ... ok
{"msg":"Crucible stats registered with UUID: ed279642-cd12-4875-9d22-1d8bffc528d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896728612Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ed279642-cd12-4875-9d22-1d8bffc528d0 has session id: 9c84cecd-63ea-4c2d-97a9-cb6429ea0ae1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896765562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896788447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896807598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896828962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896847723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.89686824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896886982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896907232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.89692529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896952555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896972539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.896993202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897011659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897032415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897050496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897071187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897089605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897110428Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897128951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"ed279642-cd12-4875-9d22-1d8bffc528d0 is now active with session: 459b81d2-7ba4-4d09-bbf5-23f5147c2925","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897148974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897169726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897188145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897208859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897227202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ed279642-cd12-4875-9d22-1d8bffc528d0 (459b81d2-7ba4-4d09-bbf5-23f5147c2925) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897247763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897266208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"job 1000 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.897284651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1000 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897309795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::three_faulted_enqueue_will_handle_flush ... ok
{"msg":"Crucible stats registered with UUID: 15d8cd85-3d92-4a9e-b443-604c8bfa1f28","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897882301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 has session id: dfa7edc9-137c-4e89-8b6a-42c071f07e00","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897916111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897937782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897956596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897977408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.897995931Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898016434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898034818Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898055336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.89807361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898094301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898112799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898133445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898151972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898172646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898198079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898225761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898245799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898266246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898284377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"15d8cd85-3d92-4a9e-b443-604c8bfa1f28 is now active with session: ab77762c-1887-4181-bbb2-afc7bf0d173c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898304205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898324482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898342521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898362773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898380974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 15d8cd85-3d92-4a9e-b443-604c8bfa1f28 (ab77762c-1887-4181-bbb2-afc7bf0d173c) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898401409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898419417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"job 1000 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.898437579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1000 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898457525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"job 1001 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.898476664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1001 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898495964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"job 1002 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.898515048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1002 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.898534466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::three_faulted_enqueue_will_handle_many_ios ... ok
{"msg":"Crucible stats registered with UUID: 34ba4809-324b-4e27-ac13-29098c033c38","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899114519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 34ba4809-324b-4e27-ac13-29098c033c38 has session id: 922d968f-9aae-41ed-a2f1-fcb4c980fc40","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899153927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.8991776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899197243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899218301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899236863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899257519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899275864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899296412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899314804Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899335745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899354508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899375354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899393713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899414258Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899432454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899453195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.89947172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899496708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899516366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"34ba4809-324b-4e27-ac13-29098c033c38 is now active with session: 75ce574e-2900-4bdd-93ad-a65d388a02a5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899536663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899557354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899575662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899596252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899614434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 34ba4809-324b-4e27-ac13-29098c033c38 (75ce574e-2900-4bdd-93ad-a65d388a02a5) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899635085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899653335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"job 1000 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.899671636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1000 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.899691395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::three_faulted_enqueue_will_handle_read ... ok
{"msg":"Crucible stats registered with UUID: 268633af-f0dd-45fe-b45e-485802948fda","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90025633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 268633af-f0dd-45fe-b45e-485802948fda has session id: 1362ed8c-38f1-40a4-bc28-a3774613e8f0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900289898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900311516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900330064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90035067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900369188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900389586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900413665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900435777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900454112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900474583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900492779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90051319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90053145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900551686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900569814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900590288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900608545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900629001Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900647118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"268633af-f0dd-45fe-b45e-485802948fda is now active with session: 1db398b3-624d-4884-ad1a-935c885a062f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900666965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900687384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90070547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900725722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900747985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 268633af-f0dd-45fe-b45e-485802948fda (1db398b3-624d-4884-ad1a-935c885a062f) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900769254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900787499Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"job 1000 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.900805596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1000 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.900825316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::three_faulted_enqueue_will_handle_write ... ok
{"msg":"Crucible stats registered with UUID: 8f5eb220-4b2e-462b-b7aa-05d0d162c535","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901395115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 8f5eb220-4b2e-462b-b7aa-05d0d162c535 has session id: 51106ee6-4468-4eab-a351-5ead15d0d68a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901428725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90145043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901469489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901490173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901508654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901529187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901547628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901568225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90158654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901607227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901625705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90165216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901672233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901692916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901711074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901731662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901750218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901770722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901788915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"8f5eb220-4b2e-462b-b7aa-05d0d162c535 is now active with session: e1047f59-8476-4547-aa31-314151aa90eb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901808678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901829049Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901847398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901867868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901886007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8f5eb220-4b2e-462b-b7aa-05d0d162c535 (e1047f59-8476-4547-aa31-314151aa90eb) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901906467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901924674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"job 1000 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.901943166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1000 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.901963112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"job 1001 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.901982314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1001 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902005892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"job 1002 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.902026389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1002 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902046071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"job 1003 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.902065563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1003 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902085028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"job 1004 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.902104168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1004 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902123594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"job 1005 skipped on all downstairs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.902142512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Enqueue job 1005 goes straight to AckReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902161878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::three_faulted_retire_skipped_some_leave_some ... ok
{"msg":"Crucible stats registered with UUID: 2a452f49-e65c-46b0-b3af-bedeb41456b1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902731949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 2a452f49-e65c-46b0-b3af-bedeb41456b1 has session id: fe4be24f-4caf-4e63-8e2a-72f42b0f7838","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902766066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902787976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902806744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902827579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902846159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902866629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902884999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902905376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902929299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902951597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902970173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.902990664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903008845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903029426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903047698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90306824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903087025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903107742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903126161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"2a452f49-e65c-46b0-b3af-bedeb41456b1 is now active with session: 6814b1ec-b4d9-4518-b268-4bd533f7ddf5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903146359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903167158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903185559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 2a452f49-e65c-46b0-b3af-bedeb41456b1 (6814b1ec-b4d9-4518-b268-4bd533f7ddf5) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903206301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903224831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::two_faulted_downstairs_can_still_read ... ok
{"msg":"Crucible stats registered with UUID: 61feadda-093e-49e4-bc24-87950912bfbf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90379417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 61feadda-093e-49e4-bc24-87950912bfbf has session id: db43a40f-88e5-4680-a866-17e54c8d7a1a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903832683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"61feadda-093e-49e4-bc24-87950912bfbf is now active with session: 8af8e09a-f3b4-4318-9cb1-8376464d7b51","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.903854997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.903886409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.903917869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.903950199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.903983Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.904015327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.90404886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.904084235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.904116072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 1001 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.904147599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.904179463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.904211278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 1001 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1001), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.90424347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_assert_reads_do_not_cause_failure_state_transition ... ok
{"msg":"Crucible stats registered with UUID: 7804a25f-01f0-40c3-9028-99afe38211f9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.904822878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 7804a25f-01f0-40c3-9028-99afe38211f9 has session id: 05d76864-f028-4f16-8fbc-bff6ad3814ee","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.904856208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"7804a25f-01f0-40c3-9028-99afe38211f9 is now active with session: 07f4e32b-00a1-4501-99f9-bd3437724390","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.904877257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.904896118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_completed_ack_read_replay ... ok
{"msg":"Crucible stats registered with UUID: 9ad61097-47b6-432f-a974-ccae159da0e2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.905513919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 9ad61097-47b6-432f-a974-ccae159da0e2 has session id: 297e80ce-ad14-41b1-a6db-e3518973f0c8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.905547478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"9ad61097-47b6-432f-a974-ccae159da0e2 is now active with session: b0320f7d-c076-4c69-b0eb-c6ddfca67a83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.905568595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.9055873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"REPLAY [0] read hash mismatch on id 1000\nExpected [Some(48a5a7677a8e488)]\nComputed [Some(ea9ca750a094f609)]\nguest_id:10 request:[ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }]\njob state:ClientData([Done, InProgress, InProgress])","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.905614834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_completed_ack_read_replay_hash_mismatch ... ok
{"msg":"Crucible stats registered with UUID: 5eee8679-0c3f-4c2d-becc-3a09b2ede674","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.906179711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 5eee8679-0c3f-4c2d-becc-3a09b2ede674 has session id: aa045a8d-8fde-421b-9172-8ade46fd0ace","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.906213185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"5eee8679-0c3f-4c2d-becc-3a09b2ede674 is now active with session: 2e9cce9b-670d-4fa1-b2a8-0670887c5706","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.906234253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.906253186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"REPLAY [1] read hash mismatch on id 1000\nExpected [Some(48a5a7677a8e488)]\nComputed [Some(ea9ca750a094f609)]\nguest_id:10 request:[ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }]\njob state:ClientData([Done, Done, InProgress])","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.906279732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_completed_ack_read_replay_two_hash_mismatch ... ok
{"msg":"Crucible stats registered with UUID: 03b74852-8a92-452e-8617-c021f4ab681e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90683881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 03b74852-8a92-452e-8617-c021f4ab681e has session id: 9f284c25-c6f4-404d-b17a-f45300a6eb68","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.906877539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"03b74852-8a92-452e-8617-c021f4ab681e is now active with session: 9f82f4d3-ee5a-443b-b078-c0f4e8a6e60e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.906899812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_completed_read_flush ... ok
{"msg":"Crucible stats registered with UUID: ee419862-b955-4190-a38c-f50331776f3d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.907545807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ee419862-b955-4190-a38c-f50331776f3d has session id: db5b7fdd-ad5f-462b-bbde-9c163d7c9303","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.907579357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"ee419862-b955-4190-a38c-f50331776f3d is now active with session: bd7e360e-589b-4752-bb9c-dda6dadd245f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.907600326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.907619037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Remove read data for 1000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.907638515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_completed_read_replay ... ok
{"msg":"Crucible stats registered with UUID: 723d9772-6b27-4b16-b803-6de21de59757","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.908200958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 723d9772-6b27-4b16-b803-6de21de59757 has session id: e078dfe2-0d9d-46f6-880b-4ee025fd2587","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.908233872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"723d9772-6b27-4b16-b803-6de21de59757 is now active with session: c1a7b7b8-7525-464b-a8b0-4412a893fc00","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90825502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.908274047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90829399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Remove read data for 1000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90831339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_completed_two_read_replay ... ok
{"msg":"Crucible stats registered with UUID: cb94f5c2-5898-4e01-9edb-5b1abb303464","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.90888673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible cb94f5c2-5898-4e01-9edb-5b1abb303464 has session id: 109b7061-9714-4123-a6c5-b268ea9811c2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.908920972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"cb94f5c2-5898-4e01-9edb-5b1abb303464 is now active with session: aee0782f-7549-40ce-a2b8-1ccc3d29aa4c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.908942113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.908960839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_completed_write_ack_ready_replay_write ... ok
{"msg":"Crucible stats registered with UUID: 0ca83e76-c347-4ce0-9c12-b33ac8c068fe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.909524365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 0ca83e76-c347-4ce0-9c12-b33ac8c068fe has session id: 4c0e6344-5857-47da-b404-54c2813d4a87","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.909557542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"0ca83e76-c347-4ce0-9c12-b33ac8c068fe is now active with session: d1e1abc7-fafe-4409-bbed-33fb6e35632d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.909578713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.909597379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"Remove AckReady for Wu/F 1000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.909616874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_completed_write_ack_ready_replay_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: a3c60a6e-7a2b-4b66-a59c-c18eb00c3ea2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.910172987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible a3c60a6e-7a2b-4b66-a59c-c18eb00c3ea2 has session id: fb2b4b2c-4dd7-43c9-acec-06cb8d080fdc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.910206407Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"a3c60a6e-7a2b-4b66-a59c-c18eb00c3ea2 is now active with session: c27c3047-c34d-42b1-bb92-6b9714feb8f1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.910227616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.910246376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_completed_write_acked_replay_write ... ok
{"msg":"Crucible stats registered with UUID: 1f9600a7-06a9-4142-a7ac-e0b3a667f0ab","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.910803445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 1f9600a7-06a9-4142-a7ac-e0b3a667f0ab has session id: 0c0ff068-7e7f-4af5-ba7e-4fbf8166332b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.910836405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"1f9600a7-06a9-4142-a7ac-e0b3a667f0ab is now active with session: 8050d4d8-814b-4881-840f-92bfd639e641","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.910857567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client re-new 1 jobs since flush 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.910876546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_completed_write_acked_replay_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: c9a76631-7a47-44a8-a263-c2d8ea009f8d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.911438984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible c9a76631-7a47-44a8-a263-c2d8ea009f8d has session id: 46bfd0c8-979b-48bb-895c-418f502063af","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.911472237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"c9a76631-7a47-44a8-a263-c2d8ea009f8d is now active with session: 8213d1cb-6d75-4e92-ae36-f0d17a1aea26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.911493306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_completed_write_flush ... ok
{"msg":"Crucible stats registered with UUID: 893e7565-f000-40e1-b286-2e46c8ccf3d7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.912131976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 893e7565-f000-40e1-b286-2e46c8ccf3d7 has session id: 715a840b-0653-48e9-be16-8bb9bacef61f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.912170393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"893e7565-f000-40e1-b286-2e46c8ccf3d7 is now active with session: 884e1daf-48b9-446f-8c24-d8353d4afaa1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.912193183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_completed_write_unwritten_flush ... ok
{"msg":"Crucible stats registered with UUID: c2df6c83-25e8-41e9-9eab-950a0052cd6e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.912770679Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible c2df6c83-25e8-41e9-9eab-950a0052cd6e has session id: 8191958a-2051-4d5b-a2d2-5c150ee08809","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.912803865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"c2df6c83-25e8-41e9-9eab-950a0052cd6e is now active with session: e9062d92-108d-4c2c-871a-821a582a987e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.912824969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_delay_completion_flush_order_write ... ok
{"msg":"Crucible stats registered with UUID: 3590da25-e844-4515-8bb6-c76ac9e53f5b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.913474006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 3590da25-e844-4515-8bb6-c76ac9e53f5b has session id: 6ae907de-bff5-4e26-a023-ff18fe667045","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.913507022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"3590da25-e844-4515-8bb6-c76ac9e53f5b is now active with session: 79619812-299a-4bf8-bc5c-836b5fb53d1c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.913528288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_delay_completion_flush_order_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: 5e51a52c-ceb1-4dab-9bdf-8141b54be0d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.914169831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 5e51a52c-ceb1-4dab-9bdf-8141b54be0d0 has session id: 2467441e-a61b-4970-bf33-34448c849376","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.9142035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"5e51a52c-ceb1-4dab-9bdf-8141b54be0d0 is now active with session: a254b3a1-de3b-4df8-9027-0ad142e8f115","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.914224988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_delay_completion_flush_write ... ok
{"msg":"Crucible stats registered with UUID: a5d20d1a-5608-4c5a-93e3-50ed6fd8be5e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.91489267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible a5d20d1a-5608-4c5a-93e3-50ed6fd8be5e has session id: 11e4bc89-11bc-4ee7-9146-37f04496fc35","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.914926263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"a5d20d1a-5608-4c5a-93e3-50ed6fd8be5e is now active with session: 9113167a-34ee-4a4f-9aa4-8f505347ff71","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.914947255Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_delay_completion_flush_write_unwritten ... ok
{"msg":"Crucible stats registered with UUID: a77ad568-3a2c-48e5-abd3-1375cab8eed8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.915600724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible a77ad568-3a2c-48e5-abd3-1375cab8eed8 has session id: 43d1f437-1fe2-4422-aad2-a2fd23f4ba1e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.915639444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"a77ad568-3a2c-48e5-abd3-1375cab8eed8 is now active with session: c8dc9abc-5bb7-4394-bca5-68f7bb96f516","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.915661949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.915690574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.915720437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_flush_one_error_then_ok ... ok
{"msg":"Crucible stats registered with UUID: e2902b2a-207c-4d49-9e41-2bd9a0901317","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.916284966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible e2902b2a-207c-4d49-9e41-2bd9a0901317 has session id: 46e0c9b1-8d0c-4caf-81f3-6694058e1861","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.916318686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"e2902b2a-207c-4d49-9e41-2bd9a0901317 is now active with session: bd66bb77-ae90-4d0e-87ad-090dc0dbd7a0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.916339553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_flush_snapshot_needs_three ... ok
{"msg":"Crucible stats registered with UUID: 80ea100b-4ab8-4801-9a99-9fecbbe4ba26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.916918874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 80ea100b-4ab8-4801-9a99-9fecbbe4ba26 has session id: 86899362-5bfd-4fbf-880c-a85b2a38c25f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.916952371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"80ea100b-4ab8-4801-9a99-9fecbbe4ba26 is now active with session: b5a47d70-3e8f-4cc4-86fe-9f6fe621db8b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.916973186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_flush_three_ok ... ok
{"msg":"Crucible stats registered with UUID: eabeb7c8-e2fb-4a2d-a1b8-cba2b9f6ee4d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.91753307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible eabeb7c8-e2fb-4a2d-a1b8-cba2b9f6ee4d has session id: b229608e-75cc-4ce7-9190-dd63eef1a9dc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.917566444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"eabeb7c8-e2fb-4a2d-a1b8-cba2b9f6ee4d is now active with session: 5caf3b84-6035-40cf-8de0-8c4715e3e015","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.917587347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.917615545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.91765367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([Error(GenericError(\"bad\")), Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.917685189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 0, work: Flush { dependencies: [], flush_number: 10, gen_number: 0, snapshot_details: None, extent_limit: None }, state: ClientData([Error(GenericError(\"bad\")), Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: Empty }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.917714729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_flush_two_errors_equals_fail ... ok
{"msg":"Crucible stats registered with UUID: 2311bf3f-63b8-458a-a536-2d3155cbfb3e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.91831282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 2311bf3f-63b8-458a-a536-2d3155cbfb3e has session id: c9d8b2e6-ee49-4ceb-ae50-2246eda892f1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.918345867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"2311bf3f-63b8-458a-a536-2d3155cbfb3e is now active with session: 41d12ee8-0343-4a8c-84dd-fc05be956162","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.918366656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_read_hash_mismatch ... ok
{"msg":"Crucible stats registered with UUID: a381e584-2bf0-43fe-b59b-137e66b44d89","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.919051291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible a381e584-2bf0-43fe-b59b-137e66b44d89 has session id: 8a6b945f-ea2f-45d5-bc32-535d0679b91d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.91908433Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"a381e584-2bf0-43fe-b59b-137e66b44d89 is now active with session: 07e37163-98b4-4267-8516-6d70b9fb4ddc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.919105289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_read_hash_mismatch_ack ... ok
{"msg":"Crucible stats registered with UUID: 3b7495fa-ba5a-44ab-9960-29bb1bf952f6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.919770358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 3b7495fa-ba5a-44ab-9960-29bb1bf952f6 has session id: f91dcaea-bed8-46d0-93fe-e24cc82ee575","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.919804193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"3b7495fa-ba5a-44ab-9960-29bb1bf952f6 is now active with session: ad1c7123-c39a-4d5f-b4c5-337132e76fe8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.919831175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_read_hash_mismatch_inside ... ok
{"msg":"Crucible stats registered with UUID: d8da0c28-5748-403b-99bf-bdcabcaad028","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.920496153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible d8da0c28-5748-403b-99bf-bdcabcaad028 has session id: 8a478a7e-be7c-4cdb-8be7-5184fad33a94","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.92052934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"d8da0c28-5748-403b-99bf-bdcabcaad028 is now active with session: 35eefb78-e525-4f1e-b6da-f8a797751acc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.920550343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_read_hash_mismatch_no_data ... ok
{"msg":"Crucible stats registered with UUID: ce168ce0-e4d8-4043-bc16-797a6bf2999a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.92120741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ce168ce0-e4d8-4043-bc16-797a6bf2999a has session id: 2fd637b5-734c-4281-8c40-e29ac96604cf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.921240588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"ce168ce0-e4d8-4043-bc16-797a6bf2999a is now active with session: 8c7ca4bd-d489-4bce-8c2f-70c287d18c48","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.921261494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_read_hash_mismatch_no_data_next ... ok
test test::up_test::work_read_hash_mismatch_third ... ok
{"msg":"Crucible stats registered with UUID: ecc5cb47-31fc-43c8-b931-d661a98739ec","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.922564615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ecc5cb47-31fc-43c8-b931-d661a98739ec has session id: 16f6c4e0-2ced-4df2-88dc-12e8d0e389cb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.922597879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"ecc5cb47-31fc-43c8-b931-d661a98739ec is now active with session: 3a5e7884-043a-418c-9d79-f6f1eea94226","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.922618675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_read_hash_mismatch_third_ack ... ok
{"msg":"Crucible stats registered with UUID: 6c66de83-c13e-4e77-88da-9fe3c346561e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.923281939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 6c66de83-c13e-4e77-88da-9fe3c346561e has session id: fd9e3094-cf7f-4855-9207-d41472dbda48","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.923315138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"6c66de83-c13e-4e77-88da-9fe3c346561e is now active with session: 24c70913-23a1-48ee-bd4c-cd255657956f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.923336121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.923366761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.923404654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.923438361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_read_one_bad_two_ok ... ok
{"msg":"Crucible stats registered with UUID: 07370796-1ca1-43e2-aab5-ac41484fa588","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.924016271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 07370796-1ca1-43e2-aab5-ac41484fa588 has session id: feda1885-36c1-4548-b570-3f7fde73ca4c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.924049808Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"07370796-1ca1-43e2-aab5-ac41484fa588 is now active with session: fb9d06e1-d83d-476d-b125-618e3c89a205","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.924070657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_read_one_ok ... ok
{"msg":"Crucible stats registered with UUID: f21a1586-42c4-4f05-942e-64105fc3f582","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.924663776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible f21a1586-42c4-4f05-942e-64105fc3f582 has session id: e4a9de7a-ea6b-4a84-8d74-0a753b5576fe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.924696929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"f21a1586-42c4-4f05-942e-64105fc3f582 is now active with session: d12b487d-03ac-4a65-ae4b-6e903b589ed7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.924717811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.92474838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.924779901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.924820183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.924854166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.924886305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.924920034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.924953903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.92498697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), Error(GenericError(\"bad\"))]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.925020423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_read_three_bad ... ok
{"msg":"Crucible stats registered with UUID: 1769b1b4-f8f6-487e-89a7-f57129f665cc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.925583417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 1769b1b4-f8f6-487e-89a7-f57129f665cc has session id: 42939798-7944-4117-9827-3c8906b7e5b0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.925617288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"1769b1b4-f8f6-487e-89a7-f57129f665cc is now active with session: 7fb7f7c5-1d41-4085-b15a-723483216d4a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.925638309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.925668966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.925700546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.925732223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.925765469Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.925797344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.925835269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_read_two_bad_one_ok ... ok
{"msg":"Crucible stats registered with UUID: 495d60dc-e511-4c6f-9bc0-cfc5be4b3b29","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.926393117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 495d60dc-e511-4c6f-9bc0-cfc5be4b3b29 has session id: 3981a62c-0154-42af-bf0f-6f429d89ef1d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.926426289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"495d60dc-e511-4c6f-9bc0-cfc5be4b3b29 is now active with session: 12ba6721-39bc-49e6-a4bb-054cedcb17cf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.926447283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 495d60dc-e511-4c6f-9bc0-cfc5be4b3b29 WARNING finish job 1000 when downstairs state:New","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.926467705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 495d60dc-e511-4c6f-9bc0-cfc5be4b3b29 WARNING finish job 1000 when downstairs state:New","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.926487683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 495d60dc-e511-4c6f-9bc0-cfc5be4b3b29 WARNING finish job 1000 when downstairs state:New","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:07.926507434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Done, InProgress, Done]), ack_status: Acked, replay: false, data: Some([ReadResponse { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\", block_contexts: [BlockContext { hash: 17241709254077376921, encryption_context: None }] }]), read_response_hashes: [Some(17241709254077376921)], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.926542613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Done, InProgress, Done]), ack_status: Acked, replay: false, data: Some([ReadResponse { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\", block_contexts: [BlockContext { hash: 17241709254077376921, encryption_context: None }] }]), read_response_hashes: [Some(17241709254077376921)], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.926579136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 1000 read error GenericError(\"bad\") DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Read { dependencies: [], requests: [ReadRequest { eid: 0, offset: Block { value: 7, shift: 9 } }] }, state: ClientData([Done, Error(GenericError(\"bad\")), Done]), ack_status: Acked, replay: false, data: Some([ReadResponse { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\", block_contexts: [BlockContext { hash: 17241709254077376921, encryption_context: None }] }]), read_response_hashes: [Some(17241709254077376921)], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.926621837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_read_two_ok_one_bad ... ok
{"msg":"Crucible stats registered with UUID: eb3ae96f-ced1-4e88-9e6c-50325f905b2f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.927193329Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible eb3ae96f-ced1-4e88-9e6c-50325f905b2f has session id: 453a16d0-30f6-448c-b7b3-061649d3a7dd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.927226746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"eb3ae96f-ced1-4e88-9e6c-50325f905b2f is now active with session: 5cf8d0a5-6ffb-4282-be61-14ff9bfc0be7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.927247785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.927281453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.9273161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.927352512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.927387386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_write_errors_are_counted ... ok
{"msg":"Crucible stats registered with UUID: b024fa9a-4d3d-406e-b7f0-fd018dbe4d87","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.927958599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible b024fa9a-4d3d-406e-b7f0-fd018dbe4d87 has session id: a1e44531-21d2-4472-9ebe-e22fe518c57f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.927991923Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"b024fa9a-4d3d-406e-b7f0-fd018dbe4d87 is now active with session: 866103d5-f460-44e5-8451-d449cad06146","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928012969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.92804622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.92808115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.928116373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.928150606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
test test::up_test::work_write_unwritten_errors_are_counted ... ok
{"msg":"Crucible stats registered with UUID: 00ccdbf3-b043-4bd8-93b0-b53aed6f525c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928729314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 00ccdbf3-b043-4bd8-93b0-b53aed6f525c has session id: 2a05c5e3-5f22-48ce-abba-7e1d0a0e7f02","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928762599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928784591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.9288034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928824261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928842796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928863097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928881281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.92890156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.92891978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928940472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928958853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.9289791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.928997229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929017663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929035804Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929056199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.92907432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929100418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929120152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"00ccdbf3-b043-4bd8-93b0-b53aed6f525c is now active with session: de9541f2-f260-432a-ba78-50547da1de7f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929140037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.929172588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.929206402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929226795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929246442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929267856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929286084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.929318835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.929352775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929378277Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929398779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) Faulted Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929420445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929438776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.929472437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Error(GenericError(\"bad\")), Error(GenericError(\"bad\")), InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.929507886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.92952816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929547615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 00ccdbf3-b043-4bd8-93b0-b53aed6f525c (de9541f2-f260-432a-ba78-50547da1de7f) Faulted Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929569916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.929588768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::work_writes_bad ... ok
{"msg":"Crucible stats registered with UUID: 33421f87-1ccb-4cfb-81de-fcfc054a60c1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.9301541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 33421f87-1ccb-4cfb-81de-fcfc054a60c1 has session id: 32fec4dc-94d8-44a6-a66f-bf241edf53a3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930187718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 33421f87-1ccb-4cfb-81de-fcfc054a60c1 (3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930215026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930235444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 33421f87-1ccb-4cfb-81de-fcfc054a60c1 (3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930256473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930274793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 33421f87-1ccb-4cfb-81de-fcfc054a60c1 (3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930295308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930313823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 33421f87-1ccb-4cfb-81de-fcfc054a60c1 (3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930334589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930352911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 33421f87-1ccb-4cfb-81de-fcfc054a60c1 (3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930373544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93039198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 33421f87-1ccb-4cfb-81de-fcfc054a60c1 (3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930412677Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93043096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 33421f87-1ccb-4cfb-81de-fcfc054a60c1 (3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930451429Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930469701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 33421f87-1ccb-4cfb-81de-fcfc054a60c1 (3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930490418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930508729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 33421f87-1ccb-4cfb-81de-fcfc054a60c1 (3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930529196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930551597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"33421f87-1ccb-4cfb-81de-fcfc054a60c1 is now active with session: 3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930572605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.930605104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.930639079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 1 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930660213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930679875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 33421f87-1ccb-4cfb-81de-fcfc054a60c1 (3b5c7131-ce19-4d7f-8c58-f07ac11ebaa3) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930702108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.930720871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_after_write_fail_is_alright ... ok
{"msg":"Crucible stats registered with UUID: 270553f6-3717-4381-b86f-5053f8a21eae","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931299722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 270553f6-3717-4381-b86f-5053f8a21eae has session id: c952298b-2110-439b-bbad-439723c8b004","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931431958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"270553f6-3717-4381-b86f-5053f8a21eae is now active with session: 4624fd44-b981-4ccb-a010-c3704c3ca4b9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931459541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931480681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931499574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931525823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931545623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931566304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931584859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931605172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931623603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931644224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931662575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931682915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931701406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931722084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931740376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93176104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931779595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931799978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931818331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93183888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931857211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 270553f6-3717-4381-b86f-5053f8a21eae (4624fd44-b981-4ccb-a010-c3704c3ca4b9) Active Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931881549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.931900915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_double_skip ... ok
{"msg":"Crucible stats registered with UUID: 90797728-3dac-480d-97b8-e1acae1f83f6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932467187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 90797728-3dac-480d-97b8-e1acae1f83f6 has session id: 7aec65a8-f9d8-4f31-bd48-00356705b043","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93250018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"90797728-3dac-480d-97b8-e1acae1f83f6 is now active with session: fdf44758-7d74-4ede-a5e5-f6f9ba3175ad","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932521329Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 90797728-3dac-480d-97b8-e1acae1f83f6 (fdf44758-7d74-4ede-a5e5-f6f9ba3175ad) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932541946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93256056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 90797728-3dac-480d-97b8-e1acae1f83f6 (fdf44758-7d74-4ede-a5e5-f6f9ba3175ad) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932581032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932599462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 90797728-3dac-480d-97b8-e1acae1f83f6 (fdf44758-7d74-4ede-a5e5-f6f9ba3175ad) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93261983Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932638254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 90797728-3dac-480d-97b8-e1acae1f83f6 (fdf44758-7d74-4ede-a5e5-f6f9ba3175ad) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932658486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93267658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 90797728-3dac-480d-97b8-e1acae1f83f6 (fdf44758-7d74-4ede-a5e5-f6f9ba3175ad) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93269731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932715723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 90797728-3dac-480d-97b8-e1acae1f83f6 (fdf44758-7d74-4ede-a5e5-f6f9ba3175ad) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932736057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93275414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 90797728-3dac-480d-97b8-e1acae1f83f6 (fdf44758-7d74-4ede-a5e5-f6f9ba3175ad) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932779969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932799896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 90797728-3dac-480d-97b8-e1acae1f83f6 (fdf44758-7d74-4ede-a5e5-f6f9ba3175ad) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932820471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932838777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 90797728-3dac-480d-97b8-e1acae1f83f6 (fdf44758-7d74-4ede-a5e5-f6f9ba3175ad) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93285911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93287742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 90797728-3dac-480d-97b8-e1acae1f83f6 (fdf44758-7d74-4ede-a5e5-f6f9ba3175ad) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932897803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.932915973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, Skipped]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.93294841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, Skipped]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.932982427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"All done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.9330028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_fail_and_skip ... ok
{"msg":"Crucible stats registered with UUID: 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93361419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc has session id: 795dc1f2-2855-49b2-92c1-cefa30100659","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933648053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc (fae73c4b-29fa-4f0a-b721-b0d0fad067e5) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93367005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933694599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc (fae73c4b-29fa-4f0a-b721-b0d0fad067e5) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933716743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933735368Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc (fae73c4b-29fa-4f0a-b721-b0d0fad067e5) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933755916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933774201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc (fae73c4b-29fa-4f0a-b721-b0d0fad067e5) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933794702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93381317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc (fae73c4b-29fa-4f0a-b721-b0d0fad067e5) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93383387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933852224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc (fae73c4b-29fa-4f0a-b721-b0d0fad067e5) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933872819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93389137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc (fae73c4b-29fa-4f0a-b721-b0d0fad067e5) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93391201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933930295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc (fae73c4b-29fa-4f0a-b721-b0d0fad067e5) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933950973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933969708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc (fae73c4b-29fa-4f0a-b721-b0d0fad067e5) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.933990486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934008954Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc is now active with session: fae73c4b-29fa-4f0a-b721-b0d0fad067e5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934033283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, Done, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.934067112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, Done, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.934100995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934122258Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934142137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 2d3b1a7f-5bd8-4cf2-ba71-8cf44b7668bc (fae73c4b-29fa-4f0a-b721-b0d0fad067e5) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934164585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934183472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_fail_past_present_future ... ok
{"msg":"Crucible stats registered with UUID: 4f4545ae-5a18-4558-9d4d-b6b789aed250","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934752563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 4f4545ae-5a18-4558-9d4d-b6b789aed250 has session id: 4800db57-e1a6-4df9-8626-09fadf459163","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934785747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 4f4545ae-5a18-4558-9d4d-b6b789aed250 (609a3918-75a6-4a97-9171-83db5dc7e1a3) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934807484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934826478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 4f4545ae-5a18-4558-9d4d-b6b789aed250 (609a3918-75a6-4a97-9171-83db5dc7e1a3) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934847405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934866077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 4f4545ae-5a18-4558-9d4d-b6b789aed250 (609a3918-75a6-4a97-9171-83db5dc7e1a3) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93488658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93491065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 4f4545ae-5a18-4558-9d4d-b6b789aed250 (609a3918-75a6-4a97-9171-83db5dc7e1a3) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934932371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.934950731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 4f4545ae-5a18-4558-9d4d-b6b789aed250 (609a3918-75a6-4a97-9171-83db5dc7e1a3) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93497142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93498985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 4f4545ae-5a18-4558-9d4d-b6b789aed250 (609a3918-75a6-4a97-9171-83db5dc7e1a3) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935010383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935028896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 4f4545ae-5a18-4558-9d4d-b6b789aed250 (609a3918-75a6-4a97-9171-83db5dc7e1a3) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93504949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935067959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 4f4545ae-5a18-4558-9d4d-b6b789aed250 (609a3918-75a6-4a97-9171-83db5dc7e1a3) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935088437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935106787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 4f4545ae-5a18-4558-9d4d-b6b789aed250 (609a3918-75a6-4a97-9171-83db5dc7e1a3) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935127285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935145528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"4f4545ae-5a18-4558-9d4d-b6b789aed250 is now active with session: 609a3918-75a6-4a97-9171-83db5dc7e1a3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935165238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.935197781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.93523575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935256908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93527637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 4f4545ae-5a18-4558-9d4d-b6b789aed250 (609a3918-75a6-4a97-9171-83db5dc7e1a3) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935298764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935317375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_fail_skips_inprogress_jobs ... ok
{"msg":"Crucible stats registered with UUID: f0fb8083-4305-44d5-b813-348c2229d463","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935887747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible f0fb8083-4305-44d5-b813-348c2229d463 has session id: 930ea42e-4697-4f96-ba50-ad58e1278fe3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935921317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f0fb8083-4305-44d5-b813-348c2229d463 (0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93594292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935961725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f0fb8083-4305-44d5-b813-348c2229d463 (0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.935982204Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936000762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f0fb8083-4305-44d5-b813-348c2229d463 (0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93607152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936105704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] f0fb8083-4305-44d5-b813-348c2229d463 (0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936133151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936152004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] f0fb8083-4305-44d5-b813-348c2229d463 (0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936177852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936197748Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] f0fb8083-4305-44d5-b813-348c2229d463 (0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936218237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936236373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] f0fb8083-4305-44d5-b813-348c2229d463 (0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936256638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936274731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] f0fb8083-4305-44d5-b813-348c2229d463 (0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93629515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936313623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] f0fb8083-4305-44d5-b813-348c2229d463 (0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936334098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936352335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"f0fb8083-4305-44d5-b813-348c2229d463 is now active with session: 0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936371934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, Done, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.936404567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, Done, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.93643818Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936458495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936482327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] f0fb8083-4305-44d5-b813-348c2229d463 (0dbd1576-efcc-4b21-ba6d-cae1dcf23f6d) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.936505588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93652424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_fail_skips_many_jobs ... ok
{"msg":"Crucible stats registered with UUID: a1a73118-3bfb-476f-959e-036eda3741ab","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937086821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible a1a73118-3bfb-476f-959e-036eda3741ab has session id: 34ff5d46-1992-4faa-88b4-15e60ffb743f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937120315Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] a1a73118-3bfb-476f-959e-036eda3741ab (81b560d5-2e28-48a8-87e9-1e1f45fdd5d8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937142295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937161435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] a1a73118-3bfb-476f-959e-036eda3741ab (81b560d5-2e28-48a8-87e9-1e1f45fdd5d8) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937182399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937201264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] a1a73118-3bfb-476f-959e-036eda3741ab (81b560d5-2e28-48a8-87e9-1e1f45fdd5d8) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937221982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937240592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] a1a73118-3bfb-476f-959e-036eda3741ab (81b560d5-2e28-48a8-87e9-1e1f45fdd5d8) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937261142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937279565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] a1a73118-3bfb-476f-959e-036eda3741ab (81b560d5-2e28-48a8-87e9-1e1f45fdd5d8) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937300365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937318984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] a1a73118-3bfb-476f-959e-036eda3741ab (81b560d5-2e28-48a8-87e9-1e1f45fdd5d8) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937339715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937358328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] a1a73118-3bfb-476f-959e-036eda3741ab (81b560d5-2e28-48a8-87e9-1e1f45fdd5d8) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937385123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937404894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] a1a73118-3bfb-476f-959e-036eda3741ab (81b560d5-2e28-48a8-87e9-1e1f45fdd5d8) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937426092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937444598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] a1a73118-3bfb-476f-959e-036eda3741ab (81b560d5-2e28-48a8-87e9-1e1f45fdd5d8) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937465162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937483492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"a1a73118-3bfb-476f-959e-036eda3741ab is now active with session: 81b560d5-2e28-48a8-87e9-1e1f45fdd5d8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937503575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.937536247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 10, work: Write { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, InProgress]), ack_status: AckReady, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.937570083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937590689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93761024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] a1a73118-3bfb-476f-959e-036eda3741ab (81b560d5-2e28-48a8-87e9-1e1f45fdd5d8) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937632811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.937651637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_fail_skips_new_jobs ... ok
{"msg":"Crucible stats registered with UUID: 5fea0624-430f-4e90-9e64-ea7c9e24dbc8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938215777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 has session id: ad865632-2d7e-4a30-9729-48987a0e96ac","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93825432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"5fea0624-430f-4e90-9e64-ea7c9e24dbc8 is now active with session: 41e514fe-864a-46fe-9eac-6b73d710d856","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938276753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 (41e514fe-864a-46fe-9eac-6b73d710d856) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938297515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938316221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 (41e514fe-864a-46fe-9eac-6b73d710d856) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938336734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938355221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 (41e514fe-864a-46fe-9eac-6b73d710d856) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938375552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938393703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 (41e514fe-864a-46fe-9eac-6b73d710d856) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938414051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938432171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 (41e514fe-864a-46fe-9eac-6b73d710d856) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938452825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938471285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 (41e514fe-864a-46fe-9eac-6b73d710d856) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938491519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938509589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 (41e514fe-864a-46fe-9eac-6b73d710d856) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938529999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938547955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 (41e514fe-864a-46fe-9eac-6b73d710d856) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938568342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938590807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 (41e514fe-864a-46fe-9eac-6b73d710d856) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938612475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93863086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 5fea0624-430f-4e90-9e64-ea7c9e24dbc8 (41e514fe-864a-46fe-9eac-6b73d710d856) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938651169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.938669282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_single_skip ... ok
{"msg":"Crucible stats registered with UUID: 830be375-01ed-4185-b9f4-a587fecd7f4c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939231962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 830be375-01ed-4185-b9f4-a587fecd7f4c has session id: a1d16d88-a9d9-41b3-aebb-43ec2fab8857","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939264826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"830be375-01ed-4185-b9f4-a587fecd7f4c is now active with session: 9761517b-640f-4e3d-a10b-ffee13c2a335","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939285982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939306671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939325534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939346174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939364528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939384882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939402979Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939423283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939441312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939461853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939486296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93950818Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939526792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939547271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93956555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939586512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939604768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939625217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.93964351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939663775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939681915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 830be375-01ed-4185-b9f4-a587fecd7f4c (9761517b-640f-4e3d-a10b-ffee13c2a335) Active Faulted Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939702504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.939720794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_unwritten_double_skip ... ok
{"msg":"Crucible stats registered with UUID: 9c388cd6-fae1-4b08-8053-810eeb837889","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940280962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 9c388cd6-fae1-4b08-8053-810eeb837889 has session id: dfbcf242-6116-407f-b4f0-6ad620d22767","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940314108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"9c388cd6-fae1-4b08-8053-810eeb837889 is now active with session: b665010c-5239-4d80-9110-d7705153361f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.94033498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 9c388cd6-fae1-4b08-8053-810eeb837889 (b665010c-5239-4d80-9110-d7705153361f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940355748Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940380391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 9c388cd6-fae1-4b08-8053-810eeb837889 (b665010c-5239-4d80-9110-d7705153361f) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940402502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.94042135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 9c388cd6-fae1-4b08-8053-810eeb837889 (b665010c-5239-4d80-9110-d7705153361f) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940441962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940460402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9c388cd6-fae1-4b08-8053-810eeb837889 (b665010c-5239-4d80-9110-d7705153361f) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940480893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.94049918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9c388cd6-fae1-4b08-8053-810eeb837889 (b665010c-5239-4d80-9110-d7705153361f) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940520003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940538414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9c388cd6-fae1-4b08-8053-810eeb837889 (b665010c-5239-4d80-9110-d7705153361f) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.94055886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940577199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 9c388cd6-fae1-4b08-8053-810eeb837889 (b665010c-5239-4d80-9110-d7705153361f) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940597979Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940616235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 9c388cd6-fae1-4b08-8053-810eeb837889 (b665010c-5239-4d80-9110-d7705153361f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940636899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940655337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 9c388cd6-fae1-4b08-8053-810eeb837889 (b665010c-5239-4d80-9110-d7705153361f) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940675921Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.94069427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 9c388cd6-fae1-4b08-8053-810eeb837889 (b665010c-5239-4d80-9110-d7705153361f) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940714569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940736576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, Skipped]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.94077068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1010, DownstairsIO { ds_id: JobId(1010), guest_id: 19, work: WriteUnwritten { dependencies: [], writes: [Write { eid: 0, offset: Block { value: 7, shift: 9 }, data: b\"\\x01\", block_context: BlockContext { hash: 0, encryption_context: None } }] }, state: ClientData([Done, InProgress, Skipped]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 7 }, ImpactedAddr { extent_id: 0, block: 7 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:07.940805139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"All done","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.940825475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_unwritten_fail_and_skip ... ok
{"msg":"Crucible stats registered with UUID: 20675499-ba9b-4cc2-afb6-7b2326fde76c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941387907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 20675499-ba9b-4cc2-afb6-7b2326fde76c has session id: f2aa5184-d49b-48c4-8cb3-d685de3e6c52","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941420969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"20675499-ba9b-4cc2-afb6-7b2326fde76c is now active with session: acddad9f-857e-4f36-a46d-d669897fa456","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941442099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 20675499-ba9b-4cc2-afb6-7b2326fde76c (acddad9f-857e-4f36-a46d-d669897fa456) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941462973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941481765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 20675499-ba9b-4cc2-afb6-7b2326fde76c (acddad9f-857e-4f36-a46d-d669897fa456) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941502384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941520946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 20675499-ba9b-4cc2-afb6-7b2326fde76c (acddad9f-857e-4f36-a46d-d669897fa456) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941541712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941560094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 20675499-ba9b-4cc2-afb6-7b2326fde76c (acddad9f-857e-4f36-a46d-d669897fa456) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941585825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941605658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 20675499-ba9b-4cc2-afb6-7b2326fde76c (acddad9f-857e-4f36-a46d-d669897fa456) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941626684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941645238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 20675499-ba9b-4cc2-afb6-7b2326fde76c (acddad9f-857e-4f36-a46d-d669897fa456) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941665709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941683974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 20675499-ba9b-4cc2-afb6-7b2326fde76c (acddad9f-857e-4f36-a46d-d669897fa456) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.94170475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941723117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 20675499-ba9b-4cc2-afb6-7b2326fde76c (acddad9f-857e-4f36-a46d-d669897fa456) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941743715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941762052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 20675499-ba9b-4cc2-afb6-7b2326fde76c (acddad9f-857e-4f36-a46d-d669897fa456) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941782488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941800584Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 20675499-ba9b-4cc2-afb6-7b2326fde76c (acddad9f-857e-4f36-a46d-d669897fa456) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941820964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:07.941839222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test test::up_test::write_unwritten_single_skip ... ok
test test_buffer_len ... ok
test test_buffer_len_after_clone ... ok
test test_buffer_len_index_overflow - should panic ... ok
test test_buffer_len_over_block_size ... ok
test test_return_iops ... ok
test volume::test::construct_file_block_io ... ok
test volume::test::test_affected_subvolumes ... ok
test volume::test::test_correct_blocks_returned_multiple_subvolumes_1 ... ok
test volume::test::test_correct_blocks_returned_multiple_subvolumes_2 ... ok
test volume::test::test_correct_blocks_returned_multiple_subvolumes_3 ... ok
test volume::test::test_correct_blocks_returned_one_subvolume ... ok
test volume::test::test_correct_blocks_returned_three_subvolumes ... ok
test volume::test::test_drop_then_recreate_test ... ok
test volume::test::test_in_memory_block_io ... ok
test volume::test::test_no_read_only_parent_for_lba_range ... ok
test volume::test::test_out_of_bounds ... ok
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.580637345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.582584667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.59432465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.594376856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.72119483Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.721264725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:08.72130214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"err:1 or:0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.721470367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 8918c699-f710-46f0-9eef-70ed6bb4c99f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.721942101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 8918c699-f710-46f0-9eef-70ed6bb4c99f has session id: d8fd5279-1bc0-4827-b800-f11efbd0b44a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.7220021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722050992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722093666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722138586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722179386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722223186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722262993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722304936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722343406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722388133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722442848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722488593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722529048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722571527Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722603214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722644416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722684502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722729703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722764017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"8918c699-f710-46f0-9eef-70ed6bb4c99f is now active with session: a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.72279881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722834771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722866568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722902415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722934547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.722970632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.72300335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.72306125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.7231031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.723136786Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.723169417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:08.725612643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:08.725688088Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.725726092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.725760809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.725799871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.725833035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.725864637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.725896472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:08.725928856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.725961375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.726019996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 98c53138-edfb-4bb6-9deb-ec8ddf307957 (9c9d5503-b2e8-4f16-b53b-34eee863d6b8) Faulted LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.72606294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.726095842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.726138446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.728453157Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.728488046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.728519052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.853917784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.853969247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.854009303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:08.854079586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test volume::test::test_parent_initialized_read_only_region_one_subvolume ... ok
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.722517589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.722599034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.722628402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.726042258Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:09.726091872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.72613223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.729860776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.729916373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:09.855340312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:09.855422956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.855482209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.855526698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.855566152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.855586897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.855605004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:09.855623359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.855641925Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.855661213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 36f6e1bc-5305-47e3-91bd-fe997b9223d6 (98b45c36-052b-4d2c-8209-f6db7df5fa0e) LiveRepair Active Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.855683004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.855701427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:09.855719935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.723919884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.724001873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.727386938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.727434993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.727475374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.727514012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:10.727548422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.727581674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:10.727626722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:10.727661417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0   Err  Done  Done  false
    2    Acked  1001    NoOp       0  Skip  Skip  Done  false
    3    Acked  1002    NoOp       0  Skip  Skip  Done  false
    4    Acked  1003  Reopen       0  Skip  Skip  Done  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          0      1      4      5
    Skipped       3      3      0      6
    Error         1      0      0      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Crucible stats registered with UUID: 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728301764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 has session id: 6e2c3b77-caac-4366-908f-9b395378044a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728358769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728396811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728433494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728468275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728498652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.72853217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728562599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728595901Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728625489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728659223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728689251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728722605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728761842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728796748Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728827576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728867817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728901938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728935304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728964828Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 is now active with session: d294a151-6695-434f-afd6-fb1516966e23","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.728997355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.729030599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.729060061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.729099232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.729136832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.729171208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.729201403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.729230801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.729262684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.729294707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.729331973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.731140529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:10.73129233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:10.7313616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.731407982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.731449648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.731490028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.731525414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.731563349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:10.73159942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.731622191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.731641776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 73140564-9a43-4886-9974-6360c31b2d26 (521e1392-21bb-441e-9084-601140ec5551) Faulted LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.731664719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.731683161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:10.731700961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: e2730307-dcf6-4644-b256-08168ac114e1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732038338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible e2730307-dcf6-4644-b256-08168ac114e1 has session id: 5d134bb6-5135-4266-a695-1be78e146b86","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732102386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732142627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732175553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732211437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732243668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732279071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732310973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732346808Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732379017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732415003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732446996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732482251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732513936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.73254961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732582019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732617935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732650185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732685649Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732723532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"e2730307-dcf6-4644-b256-08168ac114e1 is now active with session: 8d751494-4fad-49f3-9d80-50dcfbbc3677","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732759906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732794211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.73283027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732872891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732913653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732957265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.732996968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.733036193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.733075675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.733115188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.733154375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.856891467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.856944417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.856985813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.85702435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:10.8570625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857096453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:10.857130226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:10.857175986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: b256d998-7399-488e-9e7c-fec98f8e0755","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857535172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible b256d998-7399-488e-9e7c-fec98f8e0755 has session id: eca80382-6af4-4870-9732-7b0243eee46e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857589209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857628006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857661031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857697012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857729475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857765518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857797727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857833611Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857865498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857901636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857934077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.857970095Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858002557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858038471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858070601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858107661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858149002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858186672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.85822254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"b256d998-7399-488e-9e7c-fec98f8e0755 is now active with session: 87e8b462-0c21-434d-84d7-49817a811b80","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858258565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858294207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858326493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858362889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858395132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858431411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858463903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858495504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858528454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858561032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:10.858593337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test volume::test::test_parent_initialized_read_only_region_with_multiple_sub_volumes_1 ... ok
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:2  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done  Done  false
    2    Acked  1001    NoOp       0  Done  Done  Done  false
    3 NotAcked  1002    NoOp       0   New   New   New  false
    4 NotAcked  1003  Reopen       0   New   New   New  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           2      2      2      6
    Sent          0      0      0      0
    Done          2      2      2      6
    Skipped       0      0      0      0
    Error         0      0      0      0
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      2    1
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.595796735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.595873503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.595907504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.595940471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"jobs are: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.595971534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 68feb627-ebe6-44ef-be69-9d655f75db90","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596486576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 68feb627-ebe6-44ef-be69-9d655f75db90 has session id: acbf6d56-1701-497b-bbc7-f335c3a54a62","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596537087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.5965752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596608427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596644459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596676842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596712064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596744288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596779727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596811947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596847833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596879214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596917435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.596964657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597009979Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597051084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597093854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597134342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597177195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597217248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"68feb627-ebe6-44ef-be69-9d655f75db90 is now active with session: 64856612-cf41-4003-9ffd-c17c761cd49c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597256272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597299063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597339553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597383385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597423292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 68feb627-ebe6-44ef-be69-9d655f75db90 (64856612-cf41-4003-9ffd-c17c761cd49c) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597467071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597507493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597546697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597587045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597625301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.597664991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.725349844Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:11.72545276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:11.725517253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.725562468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.725595602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.725619283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.725637976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:11.725656358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.725674847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.725693873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 8918c699-f710-46f0-9eef-70ed6bb4c99f (a0a1b590-9fa5-47d1-aa6b-921e6ac1a59b) LiveRepair Active Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.725715284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.725733321Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.725751657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:11.729963364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:11.730076756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.73012464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.730161341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 8bffad4b-ac22-4ea5-a8bb-6ce70b6d15e1 (d294a151-6695-434f-afd6-fb1516966e23) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.730185649Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.73020435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.730222422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.730240524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:11.730258727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.730277026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.732751609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.732783654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.732814376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.859040302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.859096789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.859137892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:11.859179673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test volume::test::test_parent_initialized_read_only_region_with_multiple_sub_volumes_2 ... ok
test volume::test::test_parent_uninitialized_read_only_region_one_subvolume ... ok
test volume::test::test_parent_uninitialized_read_only_region_with_multiple_sub_volumes_1 ... ok
test volume::test::test_parent_uninitialized_read_only_region_with_multiple_sub_volumes_2 ... ok
test volume::test::test_read_only_parent_for_lba_range ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 < scrub_point 0. Check with your parent
block 9 < scrub_point 0. Check with your parent
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 < scrub_point 1. Check with your parent
block 9 < scrub_point 1. Check with your parent
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 < scrub_point 2. Check with your parent
block 9 < scrub_point 2. Check with your parent
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 < scrub_point 3. Check with your parent
block 9 < scrub_point 3. Check with your parent
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 < scrub_point 4. Check with your parent
block 9 < scrub_point 4. Check with your parent
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 < scrub_point 5. Check with your parent
block 9 < scrub_point 5. Check with your parent
block 0+1 <= scrub_point 6. No parent check
block 1+1 <= scrub_point 6. No parent check
block 2+1 <= scrub_point 6. No parent check
block 3+1 <= scrub_point 6. No parent check
block 4+1 <= scrub_point 6. No parent check
block 5+1 <= scrub_point 6. No parent check
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 < scrub_point 6. Check with your parent
block 9 < scrub_point 6. Check with your parent
block 0+1 <= scrub_point 7. No parent check
block 1+1 <= scrub_point 7. No parent check
block 2+1 <= scrub_point 7. No parent check
block 3+1 <= scrub_point 7. No parent check
block 4+1 <= scrub_point 7. No parent check
block 5+1 <= scrub_point 7. No parent check
block 6+1 <= scrub_point 7. No parent check
block 7 < scrub_point 7. Check with your parent
block 8 < scrub_point 7. Check with your parent
block 9 < scrub_point 7. Check with your parent
block 0+1 <= scrub_point 8. No parent check
block 1+1 <= scrub_point 8. No parent check
block 2+1 <= scrub_point 8. No parent check
block 3+1 <= scrub_point 8. No parent check
block 4+1 <= scrub_point 8. No parent check
block 5+1 <= scrub_point 8. No parent check
block 6+1 <= scrub_point 8. No parent check
block 7+1 <= scrub_point 8. No parent check
block 8 < scrub_point 8. Check with your parent
block 9 < scrub_point 8. Check with your parent
block 0+1 <= scrub_point 9. No parent check
block 1+1 <= scrub_point 9. No parent check
block 2+1 <= scrub_point 9. No parent check
block 3+1 <= scrub_point 9. No parent check
block 4+1 <= scrub_point 9. No parent check
block 5+1 <= scrub_point 9. No parent check
block 6+1 <= scrub_point 9. No parent check
block 7+1 <= scrub_point 9. No parent check
block 8+1 <= scrub_point 9. No parent check
block 9 < scrub_point 9. Check with your parent
block 0+1 <= scrub_point 10. No parent check
block 1+1 <= scrub_point 10. No parent check
block 2+1 <= scrub_point 10. No parent check
block 3+1 <= scrub_point 10. No parent check
block 4+1 <= scrub_point 10. No parent check
block 5+1 <= scrub_point 10. No parent check
block 6+1 <= scrub_point 10. No parent check
block 7+1 <= scrub_point 10. No parent check
block 8+1 <= scrub_point 10. No parent check
block 9+1 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 < scrub_point 1. Check with your parent
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 < scrub_point 2. Check with your parent
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 < scrub_point 3. Check with your parent
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 < scrub_point 4. Check with your parent
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 < scrub_point 5. Check with your parent
block 0+2 <= scrub_point 6. No parent check
block 1+2 <= scrub_point 6. No parent check
block 2+2 <= scrub_point 6. No parent check
block 3+2 <= scrub_point 6. No parent check
block 4+2 <= scrub_point 6. No parent check
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 < scrub_point 6. Check with your parent
block 0+2 <= scrub_point 7. No parent check
block 1+2 <= scrub_point 7. No parent check
block 2+2 <= scrub_point 7. No parent check
block 3+2 <= scrub_point 7. No parent check
block 4+2 <= scrub_point 7. No parent check
block 5+2 <= scrub_point 7. No parent check
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 8 < scrub_point 7. Check with your parent
block 0+2 <= scrub_point 8. No parent check
block 1+2 <= scrub_point 8. No parent check
block 2+2 <= scrub_point 8. No parent check
block 3+2 <= scrub_point 8. No parent check
block 4+2 <= scrub_point 8. No parent check
block 5+2 <= scrub_point 8. No parent check
block 6+2 <= scrub_point 8. No parent check
block 7 < scrub_point 8. Check with your parent
block 8 < scrub_point 8. Check with your parent
block 0+2 <= scrub_point 9. No parent check
block 1+2 <= scrub_point 9. No parent check
block 2+2 <= scrub_point 9. No parent check
block 3+2 <= scrub_point 9. No parent check
block 4+2 <= scrub_point 9. No parent check
block 5+2 <= scrub_point 9. No parent check
block 6+2 <= scrub_point 9. No parent check
block 7+2 <= scrub_point 9. No parent check
block 8 < scrub_point 9. Check with your parent
block 0+2 <= scrub_point 10. No parent check
block 1+2 <= scrub_point 10. No parent check
block 2+2 <= scrub_point 10. No parent check
block 3+2 <= scrub_point 10. No parent check
block 4+2 <= scrub_point 10. No parent check
block 5+2 <= scrub_point 10. No parent check
block 6+2 <= scrub_point 10. No parent check
block 7+2 <= scrub_point 10. No parent check
block 8+2 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 0+3 <= scrub_point 6. No parent check
block 1+3 <= scrub_point 6. No parent check
block 2+3 <= scrub_point 6. No parent check
block 3+3 <= scrub_point 6. No parent check
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 0+3 <= scrub_point 7. No parent check
block 1+3 <= scrub_point 7. No parent check
block 2+3 <= scrub_point 7. No parent check
block 3+3 <= scrub_point 7. No parent check
block 4+3 <= scrub_point 7. No parent check
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 0+3 <= scrub_point 8. No parent check
block 1+3 <= scrub_point 8. No parent check
block 2+3 <= scrub_point 8. No parent check
block 3+3 <= scrub_point 8. No parent check
block 4+3 <= scrub_point 8. No parent check
block 5+3 <= scrub_point 8. No parent check
block 6 < scrub_point 8. Check with your parent
block 7 < scrub_point 8. Check with your parent
block 0+3 <= scrub_point 9. No parent check
block 1+3 <= scrub_point 9. No parent check
block 2+3 <= scrub_point 9. No parent check
block 3+3 <= scrub_point 9. No parent check
block 4+3 <= scrub_point 9. No parent check
block 5+3 <= scrub_point 9. No parent check
block 6+3 <= scrub_point 9. No parent check
block 7 < scrub_point 9. Check with your parent
block 0+3 <= scrub_point 10. No parent check
block 1+3 <= scrub_point 10. No parent check
block 2+3 <= scrub_point 10. No parent check
block 3+3 <= scrub_point 10. No parent check
block 4+3 <= scrub_point 10. No parent check
block 5+3 <= scrub_point 10. No parent check
block 6+3 <= scrub_point 10. No parent check
block 7+3 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 0+4 <= scrub_point 6. No parent check
block 1+4 <= scrub_point 6. No parent check
block 2+4 <= scrub_point 6. No parent check
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 0+4 <= scrub_point 7. No parent check
block 1+4 <= scrub_point 7. No parent check
block 2+4 <= scrub_point 7. No parent check
block 3+4 <= scrub_point 7. No parent check
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 0+4 <= scrub_point 8. No parent check
block 1+4 <= scrub_point 8. No parent check
block 2+4 <= scrub_point 8. No parent check
block 3+4 <= scrub_point 8. No parent check
block 4+4 <= scrub_point 8. No parent check
block 5 < scrub_point 8. Check with your parent
block 6 < scrub_point 8. Check with your parent
block 0+4 <= scrub_point 9. No parent check
block 1+4 <= scrub_point 9. No parent check
block 2+4 <= scrub_point 9. No parent check
block 3+4 <= scrub_point 9. No parent check
block 4+4 <= scrub_point 9. No parent check
block 5+4 <= scrub_point 9. No parent check
block 6 < scrub_point 9. Check with your parent
block 0+4 <= scrub_point 10. No parent check
block 1+4 <= scrub_point 10. No parent check
block 2+4 <= scrub_point 10. No parent check
block 3+4 <= scrub_point 10. No parent check
block 4+4 <= scrub_point 10. No parent check
block 5+4 <= scrub_point 10. No parent check
block 6+4 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 0+5 <= scrub_point 6. No parent check
block 1+5 <= scrub_point 6. No parent check
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 0+5 <= scrub_point 7. No parent check
block 1+5 <= scrub_point 7. No parent check
block 2+5 <= scrub_point 7. No parent check
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 0+5 <= scrub_point 8. No parent check
block 1+5 <= scrub_point 8. No parent check
block 2+5 <= scrub_point 8. No parent check
block 3+5 <= scrub_point 8. No parent check
block 4 < scrub_point 8. Check with your parent
block 5 < scrub_point 8. Check with your parent
block 0+5 <= scrub_point 9. No parent check
block 1+5 <= scrub_point 9. No parent check
block 2+5 <= scrub_point 9. No parent check
block 3+5 <= scrub_point 9. No parent check
block 4+5 <= scrub_point 9. No parent check
block 5 < scrub_point 9. Check with your parent
block 0+5 <= scrub_point 10. No parent check
block 1+5 <= scrub_point 10. No parent check
block 2+5 <= scrub_point 10. No parent check
block 3+5 <= scrub_point 10. No parent check
block 4+5 <= scrub_point 10. No parent check
block 5+5 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0+6 <= scrub_point 6. No parent check
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 0+6 <= scrub_point 7. No parent check
block 1+6 <= scrub_point 7. No parent check
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 0+6 <= scrub_point 8. No parent check
block 1+6 <= scrub_point 8. No parent check
block 2+6 <= scrub_point 8. No parent check
block 3 < scrub_point 8. Check with your parent
block 4 < scrub_point 8. Check with your parent
block 0+6 <= scrub_point 9. No parent check
block 1+6 <= scrub_point 9. No parent check
block 2+6 <= scrub_point 9. No parent check
block 3+6 <= scrub_point 9. No parent check
block 4 < scrub_point 9. Check with your parent
block 0+6 <= scrub_point 10. No parent check
block 1+6 <= scrub_point 10. No parent check
block 2+6 <= scrub_point 10. No parent check
block 3+6 <= scrub_point 10. No parent check
block 4+6 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 0+7 <= scrub_point 7. No parent check
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 0+7 <= scrub_point 8. No parent check
block 1+7 <= scrub_point 8. No parent check
block 2 < scrub_point 8. Check with your parent
block 3 < scrub_point 8. Check with your parent
block 0+7 <= scrub_point 9. No parent check
block 1+7 <= scrub_point 9. No parent check
block 2+7 <= scrub_point 9. No parent check
block 3 < scrub_point 9. Check with your parent
block 0+7 <= scrub_point 10. No parent check
block 1+7 <= scrub_point 10. No parent check
block 2+7 <= scrub_point 10. No parent check
block 3+7 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 0+8 <= scrub_point 8. No parent check
block 1 < scrub_point 8. Check with your parent
block 2 < scrub_point 8. Check with your parent
block 0+8 <= scrub_point 9. No parent check
block 1+8 <= scrub_point 9. No parent check
block 2 < scrub_point 9. Check with your parent
block 0+8 <= scrub_point 10. No parent check
block 1+8 <= scrub_point 10. No parent check
block 2+8 <= scrub_point 10. No parent check
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 0 < scrub_point 8. Check with your parent
block 1 < scrub_point 8. Check with your parent
block 0+9 <= scrub_point 9. No parent check
block 1 < scrub_point 9. Check with your parent
block 0+9 <= scrub_point 10. No parent check
block 1+9 <= scrub_point 10. No parent check
test volume::test::test_scrub_point_subvolume_equal ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
test volume::test::test_scrub_point_subvolume_smaller ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 6. No parent check
block 1+1 <= scrub_point 6. No parent check
block 2+1 <= scrub_point 6. No parent check
block 3+1 <= scrub_point 6. No parent check
block 4+1 <= scrub_point 6. No parent check
block 5+1 <= scrub_point 6. No parent check
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 7. No parent check
block 1+1 <= scrub_point 7. No parent check
block 2+1 <= scrub_point 7. No parent check
block 3+1 <= scrub_point 7. No parent check
block 4+1 <= scrub_point 7. No parent check
block 5+1 <= scrub_point 7. No parent check
block 6+1 <= scrub_point 7. No parent check
block 7 < scrub_point 7. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 8. No parent check
block 1+1 <= scrub_point 8. No parent check
block 2+1 <= scrub_point 8. No parent check
block 3+1 <= scrub_point 8. No parent check
block 4+1 <= scrub_point 8. No parent check
block 5+1 <= scrub_point 8. No parent check
block 6+1 <= scrub_point 8. No parent check
block 7+1 <= scrub_point 8. No parent check
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 6. No parent check
block 1+2 <= scrub_point 6. No parent check
block 2+2 <= scrub_point 6. No parent check
block 3+2 <= scrub_point 6. No parent check
block 4+2 <= scrub_point 6. No parent check
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 7. No parent check
block 1+2 <= scrub_point 7. No parent check
block 2+2 <= scrub_point 7. No parent check
block 3+2 <= scrub_point 7. No parent check
block 4+2 <= scrub_point 7. No parent check
block 5+2 <= scrub_point 7. No parent check
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 8. No parent check
block 1+2 <= scrub_point 8. No parent check
block 2+2 <= scrub_point 8. No parent check
block 3+2 <= scrub_point 8. No parent check
block 4+2 <= scrub_point 8. No parent check
block 5+2 <= scrub_point 8. No parent check
block 6+2 <= scrub_point 8. No parent check
block 7 < scrub_point 8. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 0+3 <= scrub_point 6. No parent check
block 1+3 <= scrub_point 6. No parent check
block 2+3 <= scrub_point 6. No parent check
block 3+3 <= scrub_point 6. No parent check
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 0+3 <= scrub_point 7. No parent check
block 1+3 <= scrub_point 7. No parent check
block 2+3 <= scrub_point 7. No parent check
block 3+3 <= scrub_point 7. No parent check
block 4+3 <= scrub_point 7. No parent check
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 0+3 <= scrub_point 8. No parent check
block 1+3 <= scrub_point 8. No parent check
block 2+3 <= scrub_point 8. No parent check
block 3+3 <= scrub_point 8. No parent check
block 4+3 <= scrub_point 8. No parent check
block 5+3 <= scrub_point 8. No parent check
block 6 < scrub_point 8. Check with your parent
block 7 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 0+4 <= scrub_point 6. No parent check
block 1+4 <= scrub_point 6. No parent check
block 2+4 <= scrub_point 6. No parent check
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 0+4 <= scrub_point 7. No parent check
block 1+4 <= scrub_point 7. No parent check
block 2+4 <= scrub_point 7. No parent check
block 3+4 <= scrub_point 7. No parent check
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 0+4 <= scrub_point 8. No parent check
block 1+4 <= scrub_point 8. No parent check
block 2+4 <= scrub_point 8. No parent check
block 3+4 <= scrub_point 8. No parent check
block 4+4 <= scrub_point 8. No parent check
block 5 < scrub_point 8. Check with your parent
block 6 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 0+5 <= scrub_point 6. No parent check
block 1+5 <= scrub_point 6. No parent check
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 0+5 <= scrub_point 7. No parent check
block 1+5 <= scrub_point 7. No parent check
block 2+5 <= scrub_point 7. No parent check
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 0+5 <= scrub_point 8. No parent check
block 1+5 <= scrub_point 8. No parent check
block 2+5 <= scrub_point 8. No parent check
block 3+5 <= scrub_point 8. No parent check
block 4 < scrub_point 8. Check with your parent
block 5 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0+6 <= scrub_point 6. No parent check
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 0+6 <= scrub_point 7. No parent check
block 1+6 <= scrub_point 7. No parent check
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 0+6 <= scrub_point 8. No parent check
block 1+6 <= scrub_point 8. No parent check
block 2+6 <= scrub_point 8. No parent check
block 3 < scrub_point 8. Check with your parent
block 4 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 0+7 <= scrub_point 7. No parent check
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 0+7 <= scrub_point 8. No parent check
block 1+7 <= scrub_point 8. No parent check
block 2 < scrub_point 8. Check with your parent
block 3 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 0+8 <= scrub_point 8. No parent check
block 1 < scrub_point 8. Check with your parent
block 2 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 0 < scrub_point 8. Check with your parent
block 1 < scrub_point 8. Check with your parent
test volume::test::test_scrub_point_two_subvolume_equal ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 14 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 13 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 12 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 11 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 10 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
test volume::test::test_scrub_point_two_subvolume_smaller_1 ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 9 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 8 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 7 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 6 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 > parent 5. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
test volume::test::test_scrub_point_two_subvolume_smaller_2 ... ok
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 1. No parent check
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 2. No parent check
block 1+1 <= scrub_point 2. No parent check
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 3. No parent check
block 1+1 <= scrub_point 3. No parent check
block 2+1 <= scrub_point 3. No parent check
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 4. No parent check
block 1+1 <= scrub_point 4. No parent check
block 2+1 <= scrub_point 4. No parent check
block 3+1 <= scrub_point 4. No parent check
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 5. No parent check
block 1+1 <= scrub_point 5. No parent check
block 2+1 <= scrub_point 5. No parent check
block 3+1 <= scrub_point 5. No parent check
block 4+1 <= scrub_point 5. No parent check
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 6. No parent check
block 1+1 <= scrub_point 6. No parent check
block 2+1 <= scrub_point 6. No parent check
block 3+1 <= scrub_point 6. No parent check
block 4+1 <= scrub_point 6. No parent check
block 5+1 <= scrub_point 6. No parent check
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 7. No parent check
block 1+1 <= scrub_point 7. No parent check
block 2+1 <= scrub_point 7. No parent check
block 3+1 <= scrub_point 7. No parent check
block 4+1 <= scrub_point 7. No parent check
block 5+1 <= scrub_point 7. No parent check
block 6+1 <= scrub_point 7. No parent check
block 7 < scrub_point 7. Check with your parent
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0+1 <= scrub_point 8. No parent check
block 1+1 <= scrub_point 8. No parent check
block 2+1 <= scrub_point 8. No parent check
block 3+1 <= scrub_point 8. No parent check
block 4+1 <= scrub_point 8. No parent check
block 5+1 <= scrub_point 8. No parent check
block 6+1 <= scrub_point 8. No parent check
block 7+1 <= scrub_point 8. No parent check
block 8 > parent 8. Go to SubVolume
block 9 > parent 8. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 2. No parent check
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 3. No parent check
block 1+2 <= scrub_point 3. No parent check
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 4. No parent check
block 1+2 <= scrub_point 4. No parent check
block 2+2 <= scrub_point 4. No parent check
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 5. No parent check
block 1+2 <= scrub_point 5. No parent check
block 2+2 <= scrub_point 5. No parent check
block 3+2 <= scrub_point 5. No parent check
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 6. No parent check
block 1+2 <= scrub_point 6. No parent check
block 2+2 <= scrub_point 6. No parent check
block 3+2 <= scrub_point 6. No parent check
block 4+2 <= scrub_point 6. No parent check
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 7. No parent check
block 1+2 <= scrub_point 7. No parent check
block 2+2 <= scrub_point 7. No parent check
block 3+2 <= scrub_point 7. No parent check
block 4+2 <= scrub_point 7. No parent check
block 5+2 <= scrub_point 7. No parent check
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0+2 <= scrub_point 8. No parent check
block 1+2 <= scrub_point 8. No parent check
block 2+2 <= scrub_point 8. No parent check
block 3+2 <= scrub_point 8. No parent check
block 4+2 <= scrub_point 8. No parent check
block 5+2 <= scrub_point 8. No parent check
block 6+2 <= scrub_point 8. No parent check
block 7 < scrub_point 8. Check with your parent
block 8 > parent 8. Go to SubVolume
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 7 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 7 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 7 < scrub_point 2. Check with your parent
block 0+3 <= scrub_point 3. No parent check
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 7 < scrub_point 3. Check with your parent
block 0+3 <= scrub_point 4. No parent check
block 1+3 <= scrub_point 4. No parent check
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 7 < scrub_point 4. Check with your parent
block 0+3 <= scrub_point 5. No parent check
block 1+3 <= scrub_point 5. No parent check
block 2+3 <= scrub_point 5. No parent check
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 7 < scrub_point 5. Check with your parent
block 0+3 <= scrub_point 6. No parent check
block 1+3 <= scrub_point 6. No parent check
block 2+3 <= scrub_point 6. No parent check
block 3+3 <= scrub_point 6. No parent check
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 7 < scrub_point 6. Check with your parent
block 0+3 <= scrub_point 7. No parent check
block 1+3 <= scrub_point 7. No parent check
block 2+3 <= scrub_point 7. No parent check
block 3+3 <= scrub_point 7. No parent check
block 4+3 <= scrub_point 7. No parent check
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 7 < scrub_point 7. Check with your parent
block 0+3 <= scrub_point 8. No parent check
block 1+3 <= scrub_point 8. No parent check
block 2+3 <= scrub_point 8. No parent check
block 3+3 <= scrub_point 8. No parent check
block 4+3 <= scrub_point 8. No parent check
block 5+3 <= scrub_point 8. No parent check
block 6 < scrub_point 8. Check with your parent
block 7 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 6 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 6 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 6 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 6 < scrub_point 3. Check with your parent
block 0+4 <= scrub_point 4. No parent check
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 6 < scrub_point 4. Check with your parent
block 0+4 <= scrub_point 5. No parent check
block 1+4 <= scrub_point 5. No parent check
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 6 < scrub_point 5. Check with your parent
block 0+4 <= scrub_point 6. No parent check
block 1+4 <= scrub_point 6. No parent check
block 2+4 <= scrub_point 6. No parent check
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 6 < scrub_point 6. Check with your parent
block 0+4 <= scrub_point 7. No parent check
block 1+4 <= scrub_point 7. No parent check
block 2+4 <= scrub_point 7. No parent check
block 3+4 <= scrub_point 7. No parent check
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 6 < scrub_point 7. Check with your parent
block 0+4 <= scrub_point 8. No parent check
block 1+4 <= scrub_point 8. No parent check
block 2+4 <= scrub_point 8. No parent check
block 3+4 <= scrub_point 8. No parent check
block 4+4 <= scrub_point 8. No parent check
block 5 < scrub_point 8. Check with your parent
block 6 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 5 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 5 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 5 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 5 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 5 < scrub_point 4. Check with your parent
block 0+5 <= scrub_point 5. No parent check
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 5 < scrub_point 5. Check with your parent
block 0+5 <= scrub_point 6. No parent check
block 1+5 <= scrub_point 6. No parent check
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 5 < scrub_point 6. Check with your parent
block 0+5 <= scrub_point 7. No parent check
block 1+5 <= scrub_point 7. No parent check
block 2+5 <= scrub_point 7. No parent check
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 5 < scrub_point 7. Check with your parent
block 0+5 <= scrub_point 8. No parent check
block 1+5 <= scrub_point 8. No parent check
block 2+5 <= scrub_point 8. No parent check
block 3+5 <= scrub_point 8. No parent check
block 4 < scrub_point 8. Check with your parent
block 5 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 4 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 4 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 4 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 4 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 4 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 4 < scrub_point 5. Check with your parent
block 0+6 <= scrub_point 6. No parent check
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 4 < scrub_point 6. Check with your parent
block 0+6 <= scrub_point 7. No parent check
block 1+6 <= scrub_point 7. No parent check
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 4 < scrub_point 7. Check with your parent
block 0+6 <= scrub_point 8. No parent check
block 1+6 <= scrub_point 8. No parent check
block 2+6 <= scrub_point 8. No parent check
block 3 < scrub_point 8. Check with your parent
block 4 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 3 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 3 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 3 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 3 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 3 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 3 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 3 < scrub_point 6. Check with your parent
block 0+7 <= scrub_point 7. No parent check
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 3 < scrub_point 7. Check with your parent
block 0+7 <= scrub_point 8. No parent check
block 1+7 <= scrub_point 8. No parent check
block 2 < scrub_point 8. Check with your parent
block 3 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 2 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 2 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 2 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 2 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 2 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 2 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 2 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 2 < scrub_point 7. Check with your parent
block 0+8 <= scrub_point 8. No parent check
block 1 < scrub_point 8. Check with your parent
block 2 < scrub_point 8. Check with your parent
block 0 < scrub_point 0. Check with your parent
block 1 < scrub_point 0. Check with your parent
block 0 < scrub_point 1. Check with your parent
block 1 < scrub_point 1. Check with your parent
block 0 < scrub_point 2. Check with your parent
block 1 < scrub_point 2. Check with your parent
block 0 < scrub_point 3. Check with your parent
block 1 < scrub_point 3. Check with your parent
block 0 < scrub_point 4. Check with your parent
block 1 < scrub_point 4. Check with your parent
block 0 < scrub_point 5. Check with your parent
block 1 < scrub_point 5. Check with your parent
block 0 < scrub_point 6. Check with your parent
block 1 < scrub_point 6. Check with your parent
block 0 < scrub_point 7. Check with your parent
block 1 < scrub_point 7. Check with your parent
block 0 < scrub_point 8. Check with your parent
block 1 < scrub_point 8. Check with your parent
test volume::test::test_scrub_point_two_subvolume_smaller_3 ... ok
test volume::test::test_single_block ... ok
test volume::test::test_single_sub_volume_lba_coverage ... ok
test volume::test::test_single_sub_volume_lba_coverage_with_offset ... ok
test volume::test::test_three_layers ... ok
test volume::test::test_volume_size ... ok
test volume::test::test_volume_with_only_read_only_parent ... ok
test volume::test::test_write_unwritten_to_volume_with_only_read_only_parent ... ok
test volume::test::test_writing_to_volume_with_only_read_only_parent ... ok
Sep 22 23:15:12.342 INFO Test replacement of CID 0
Sep 22 23:15:12.342 INFO replace 127.0.0.1:5555 with 127.0.0.1:8888
Sep 22 23:15:12.342 INFO Test replacement of CID 1
Sep 22 23:15:12.342 INFO replace 127.0.0.1:6666 with 127.0.0.1:8888
Sep 22 23:15:12.342 INFO Test replacement of CID 2
Sep 22 23:15:12.342 INFO replace 127.0.0.1:7777 with 127.0.0.1:8888
test volume::test::volume_replace_basic ... ok
test volume::test::volume_replace_drop_rop ... ok
test volume::test::volume_replace_mismatch_opts_cert_pem ... ok
test volume::test::volume_replace_mismatch_opts_control ... ok
test volume::test::volume_replace_mismatch_opts_flush_timeout ... ok
test volume::test::volume_replace_mismatch_opts_id ... ok
test volume::test::volume_replace_mismatch_opts_key ... ok
test volume::test::volume_replace_mismatch_opts_key_pem ... ok
test volume::test::volume_replace_mismatch_opts_lossy ... ok
test volume::test::volume_replace_mismatch_opts_read_only ... ok
test volume::test::volume_replace_mismatch_opts_root_cert ... ok
test volume::test::volume_replace_mismatch_sv_bpe ... ok
test volume::test::volume_replace_mismatch_sv_bs ... ok
test volume::test::volume_replace_mismatch_sv_ec ... ok
test volume::test::volume_replace_mismatch_vblock ... ok
test volume::test::volume_replace_mismatch_vid ... ok
test volume::test::volume_replace_mismatch_vrop ... ok
test volume::test::volume_replace_rop ... ok
test volume::test::volume_replace_self ... ok
test volume::test::volume_vcr_no_target ... ok
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.597049145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.597100264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.597123159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.731463895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:12.731542116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.731569038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.734134664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.734177007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:12.859485405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:12.859553605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.859575323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.859594904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.859616959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.859635365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.859653416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:12.859671478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.859689886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.859708858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] b256d998-7399-488e-9e7c-fec98f8e0755 (87e8b462-0c21-434d-84d7-49817a811b80) Faulted LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.859730377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.859748578Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:12.859767062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.598407403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.598468832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.726785785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.72682494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:13.726844236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"err:2 or:0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.726998303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727434169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b has session id: e6f4a709-5395-4d56-8e67-91d7f07e4ff7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727467694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.72748968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727508818Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727529355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727547836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727568124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727586327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727606501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727624777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727645402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.72766375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727684321Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727702429Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727730743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727750044Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727770783Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727789081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727809406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.72782761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b is now active with session: ff123089-f6f2-4dc3-8c57-434e173b3d67","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727847387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727867648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727885865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727906595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727924917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727945656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727964133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.727982265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.728000813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.728018784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.728036943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.732780011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.732842365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.732884877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.732925061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:13.73296356Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.73299761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:13.733031228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759----------------------------------------------------------------
}
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
{"msg":GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
"RE:0 Bailing with error","v":0,"name":"crucible","    1    Acked  1000  FClose       0level":40  Done   Err  Done  false
    2    Acked  1001    NoOp       0  Done  Skip  Done  false
    3    Acked  1002    NoOp       0,"  Donetime  Skip"  Done:  false"
2023-09-22T23:15:13.733079694Z",    4    Acked  1003  Reopen       0"  Done  Skiphostname  Done"  false:
"   STATES      DS:0   DS:1   DS:2   TOTAL
ip-10-150-1-55.us-west-2.compute.internal"    New        ,"pid   0   ":   0   4759   0   }   0

    Sent          0      0      0      0
    Done          4      0      4      8
    Skipped       0      3      0      3
    Error         0      1      0      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Crucible stats registered with UUID: 342ffdcc-1be9-41e2-9df4-d887f80de943","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733531093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 342ffdcc-1be9-41e2-9df4-d887f80de943 has session id: 3681e62b-cf90-4959-8dd8-3b7e79ddb73f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733563731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733585474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733604123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733624703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733643033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733663055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733681186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733701376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733725767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733747676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733766216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733786465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733804485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733824554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733842489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733862928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733881006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733901238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733919253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"342ffdcc-1be9-41e2-9df4-d887f80de943 is now active with session: 36f75453-9462-4780-abce-7f7f3c31d228","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.73393924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733959754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733977683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.733998074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.734016355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.734036843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.734059377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.734078376Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.734096953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.734115011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.734133318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.735423519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:13.735541176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:13.735581641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.735602659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.735622261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] e2730307-dcf6-4644-b256-08168ac114e1 (8d751494-4fad-49f3-9d80-50dcfbbc3677) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.735644275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.735662846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.735681277Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:13.735699576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:13.735717341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 56fedc7f-6649-4d9c-8d0a-ac22f65da705","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.73602083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 56fedc7f-6649-4d9c-8d0a-ac22f65da705 has session id: 53c11a87-8b8f-4c6f-84ef-36f22a19d7c6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736061466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736085303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736104298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736125018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736143405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736163955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736182079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736202301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736220425Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736241016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736259413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736279761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736297702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736317951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736335947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736356345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.7363747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736399799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736418984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"56fedc7f-6649-4d9c-8d0a-ac22f65da705 is now active with session: 38051f6c-52ee-44a3-80ec-3f0d3c3a61b5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736438839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736459304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736477351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736498038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736516383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736537109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736555593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736573618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736592247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736610397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.736628806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.860894086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.860959284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.860980751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.860999289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:13.861017661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861036437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:13.861054858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:13.861081595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: e2a24436-8a58-49c3-b253-d7ce5f8c2715","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861526655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible e2a24436-8a58-49c3-b253-d7ce5f8c2715 has session id: 36f3d398-f991-4e44-9702-12dd51e5824b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861559676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861581636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861600533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861621214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861639634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861659891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861678032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861698197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861716355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861736972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861755153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861775594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861793623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861814016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861831939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861858999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861878819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861899381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861917516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"e2a24436-8a58-49c3-b253-d7ce5f8c2715 is now active with session: 1d7b944a-0284-455a-8b0b-72a46a80d72a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861937419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861957728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861975717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.861996184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.862014381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.862034831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.862053161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.86207104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.862089967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.862107988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:13.862126475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.729008037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.729060575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.729081985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:14.734102342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(2), repair_downstairs: [ClientId(1)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:14.734160641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.734185251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.734205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.734227271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.734245589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.734263395Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.734281234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:14.734299415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.73431774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.734337014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 342ffdcc-1be9-41e2-9df4-d887f80de943 (36f75453-9462-4780-abce-7f7f3c31d228) Active LiveRepair Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.734358244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.734376419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.734394886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.736565674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.73660252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.736631712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.863169136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.863211445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.863230993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:14.863251696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.729484962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.729539634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.735622493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:15.735655167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.735674841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.737994616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.738031341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:15.863557938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:15.863617301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.863638314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.86365768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] e2a24436-8a58-49c3-b253-d7ce5f8c2715 (1d7b944a-0284-455a-8b0b-72a46a80d72a) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.863679795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.863710223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.86372988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:15.863748387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:15.863766887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:2  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done  Done  false
    2    Acked  1001    NoOp       0  Done  Done  Done  false
    3 NotAcked  1002    NoOp       0   New   New   New  false
    4 NotAcked  1003  Reopen       0   New   New   New  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           2      2      2      6
    Sent          0      0      0      0
    Done          2      2      2      6
    Skipped       0      0      0      0
    Error         0      0      0      0
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      2    1
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.598854705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.598910864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.598931086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.598949892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"jobs are: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.599043295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_extent_no_action_all ... ok
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.730787054Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:16.730849702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:16.730881658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.730919598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.730950615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.730973947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.730992323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:16.73101063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.731029622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.731048966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] f006ef8c-c009-4b2e-bcc5-fdf6b7dadb1b (ff123089-f6f2-4dc3-8c57-434e173b3d67) Faulted LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.731070784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.731089245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.73110755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.735970228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736028843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736072253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736113519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
,"time":"GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
2023-09-22T23:15:16.736153514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"    1    Acked  1000  FClose       0,"  Done  Donepid"   Err:  false4759
}
    2    Acked  1001    NoOp       0  Done  Skip  Skip  false
{    3    Acked  1002    NoOp       0  Done  Skip  Skip  false
"    4    Acked  1003  Reopen       0msg"  Done:  Skip  Skip"  false
   STATES      DS:0   DS:1   DS:2   TOTAL
RE:0 Wait for result from reopen command 1003:4"    New        ,   0   "   0   v":   0   0   0
,    Sent       "   0   name"   0   :   0   "   0
    Done       crucible"   4      1   ,   0   "   5
    Skipped    level"   0   :   3   30   3      6
    Error         0      0      1      1
Last Flush: 0 0 0 
Downstairs last five completed:
,"time"Upstairs last five completed:  :"    42023-09-22T23:15:16.736233281Z"    3    2    1
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:16.736282387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:16.736322939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 8aebef52-94cd-4f20-8828-2b4a72214191","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.73680114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 8aebef52-94cd-4f20-8828-2b4a72214191 has session id: 4dca0879-c48b-435e-bfa8-ff047a61d6cf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736836922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.73685966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736878911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736899644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736918134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736938503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736956563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736976944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.736995167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737015701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737033996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737054443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737072471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737092732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.73711076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737131439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737156067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737178159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737196521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"8aebef52-94cd-4f20-8828-2b4a72214191 is now active with session: 217e8b67-8753-4bb1-be70-961d7790fe1e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737216231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737236665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737254684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737275293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737293758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737314242Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737332683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737350792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737369439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737387405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.737405879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739283695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:16.739398184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:16.739438615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739466682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739487855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739510233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739528841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739547324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:16.73956543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739584106Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739603429Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 56fedc7f-6649-4d9c-8d0a-ac22f65da705 (38051f6c-52ee-44a3-80ec-3f0d3c3a61b5) Active LiveRepair Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739626048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739644953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:16.739662895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 889fe5fc-0632-4e64-8f28-45e2de5accae","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.739995128Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 889fe5fc-0632-4e64-8f28-45e2de5accae has session id: 61f44ac6-6799-447c-a841-782f5bb4b649","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740028481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.74005034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740069162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740089572Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740107977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740134415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740154102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.74017498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740193312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740213856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740232218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740252505Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740270673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740291101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.74030925Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740329666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740347786Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740368136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740386204Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"889fe5fc-0632-4e64-8f28-45e2de5accae is now active with session: 0cc985d8-8a6a-47f7-b1fd-cb9776e7451b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740405758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740426206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740444143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740468964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740488308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740509059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740527416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740545535Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740564267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740582336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.740600591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.864955799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865011246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865031752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865050068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:16.865068434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865087262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:16.865105519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:16.865123322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 01994c6f-c2d5-493f-a8f0-5367919f7ab3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865652716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 01994c6f-c2d5-493f-a8f0-5367919f7ab3 has session id: 547be00f-e88f-4516-bfaa-248097069cea","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865686171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.86570813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865727372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865759564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865779583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865800177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865818304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865838626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865856852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865877504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865895887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865916201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865934341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865954913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865973122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.865993644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866011992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866032339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866050531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"01994c6f-c2d5-493f-a8f0-5367919f7ab3 is now active with session: 7327196b-8480-4c3d-8d38-567327aabf7f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866070504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866096276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866115547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866136287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866154743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.86617538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866194067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866212097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866230835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866248946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:16.866267313Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:17.738404424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:17.73847296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.738495043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.738514506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.738536327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.738565461Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.738584643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.738602926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:17.738621065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.73863945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.738658756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 8aebef52-94cd-4f20-8828-2b4a72214191 (217e8b67-8753-4bb1-be70-961d7790fe1e) Faulted Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.738680384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.73869863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.73871719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.740545284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.740581644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.740603292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.867207627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.867261445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.86728059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:17.867301139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.581601488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.583549503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.732199689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.732253517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:18.732272513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"err:0 or:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.732414307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: cb3ee61f-889b-4ca8-a683-5edf45aa3b9f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.73285132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible cb3ee61f-889b-4ca8-a683-5edf45aa3b9f has session id: 92cb7af1-472f-4178-9c2c-4b85812af232","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.732884142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.732906418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.732925265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.73294598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.732964734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.732984997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733003239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733023633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733041832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733062434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733080789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.73310112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733119256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733139746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733157756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733178079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733203301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.73322479Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733242999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"cb3ee61f-889b-4ca8-a683-5edf45aa3b9f is now active with session: 15d39748-d69b-4979-93e2-ab7205c3ae88","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733262746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.73328281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733300861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733321347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733339574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733360123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733378607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733396494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733415127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733433364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.733451794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.739874461Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:18.739908245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.739927682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.741917535Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.741963116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:18.868719403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:18.868786423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.86880788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.868827767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.868877904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.868897853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.868916232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:18.868934565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.868952985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.86897269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 01994c6f-c2d5-493f-a8f0-5367919f7ab3 (7327196b-8480-4c3d-8d38-567327aabf7f) Active LiveRepair Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.868994627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.869013097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:18.869031833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.733561933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.733614675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.733646955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.7412869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.741321944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.741341537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.741359108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:19.741377355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.741395762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:19.741414032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:19.741431752Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0   Err  Done  Done  false
    2    Acked  1001    NoOp       0  Skip  Done  Skip  false
    3    Acked  1002    NoOp       0  Skip  Done  Skip  false
    4    Acked  1003  Reopen       0  Skip  Done  Skip  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          0      4      1      5
    Skipped       3      0      3      6
    Error         1      0      0      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Crucible stats registered with UUID: 8192eba5-97c6-48ab-8071-94d8c33e981b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742075718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 8192eba5-97c6-48ab-8071-94d8c33e981b has session id: db40d87f-3f0f-4425-b8f0-ae0a1d863219","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742111101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742133646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742152683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742173645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742192219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742212628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742238613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742260684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.74227922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742300005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.74231832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742338887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742357302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742377949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742396294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742417063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742435566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742456085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742474523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"8192eba5-97c6-48ab-8071-94d8c33e981b is now active with session: 36b7cbe2-43d9-4803-b4a9-182785cca042","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742494631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.74251502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742532991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742553491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.74257191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742597118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742616848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742635059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742653837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.742672224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.74269061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743305522Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:19.743376959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:19.743411808Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743432502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743452112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743474402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743492974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743511518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:19.743529724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743555773Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743576703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 889fe5fc-0632-4e64-8f28-45e2de5accae (0cc985d8-8a6a-47f7-b1fd-cb9776e7451b) Faulted Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743598705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743617182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:19.743635087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 1be45e4a-5a68-4e53-8c36-ebf527273e3d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.74393811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 1be45e4a-5a68-4e53-8c36-ebf527273e3d has session id: 34f73e1e-9954-45b5-9b2a-d076dda2cd1d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.74397152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.743993928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744012946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744033629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744052018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744072324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744090525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744110638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744128664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744149268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744167394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.74418772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744212557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744234192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744252486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.74427306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744291388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744311957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744330076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"1be45e4a-5a68-4e53-8c36-ebf527273e3d is now active with session: 1cc0e6a6-7f15-4d27-989e-892a7dc5f1be","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744349843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744370317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744388531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744409172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744427506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744448264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.74446667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744484759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744503399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744521633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.744539873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870128021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870182854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870203384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870221707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:19.870240197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870259Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:19.870277862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:19.870295822Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: cf578269-121b-47cb-9294-332f82a54c8d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870827171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible cf578269-121b-47cb-9294-332f82a54c8d has session id: e7759d74-a4ba-467d-a028-464f8d9cc295","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870861198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.87088363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870902821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870923687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870942218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870962988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.870981557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871001667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871019719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871040318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871069554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871091264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871109482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871129744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871147755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871168331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.87118643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871206911Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871224848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"cf578269-121b-47cb-9294-332f82a54c8d is now active with session: b7ea8e7a-8c29-4b17-900a-18bdd32b26b0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871244551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871264598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871282751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871303239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871321513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871342127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871360381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871378219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871396735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871419274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:19.871438683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.734942184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.734996568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:20.743665141Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:20.743712936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.743734486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.743753949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.743776054Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.743794432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.74381223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.743830537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:20.743848848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.743867349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.743886608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 8192eba5-97c6-48ab-8071-94d8c33e981b (36b7cbe2-43d9-4803-b4a9-182785cca042) Active Faulted LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.743908042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.743936663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.743956524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.744492022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.744528819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.74455101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.871427257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.871481909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.871501666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:20.871522402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.735251587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:21.735320224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:21.735354038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.735374837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.735394235Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] cb3ee61f-889b-4ca8-a683-5edf45aa3b9f (15d39748-d69b-4979-93e2-ab7205c3ae88) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.735416538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.735435081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:21.735464836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.735484899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.745107605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:21.745140438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.745160017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.745853749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.745890871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:21.872842301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:21.872906728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.872928087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.872947383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.872969463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.872987802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.873005805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:21.873023885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.873042137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.873061154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] cf578269-121b-47cb-9294-332f82a54c8d (b7ea8e7a-8c29-4b17-900a-18bdd32b26b0) Faulted Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.873092108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.873111736Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:21.873130164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.746431919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.746483951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.746503438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.74652143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:22.74654011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.746558456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:22.746576895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:22.746594512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done   Err  Done  false
    2    Acked  1001    NoOp       0  Done  Skip  Skip  false
    3    Acked  1002    NoOp       0  Done  Skip  Skip  false
    4    Acked  1003  Reopen       0  Done  Skip  Skip  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          4      0      1      5
    Skipped       0      3      3      6
    Error         0      1      0      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747159846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\"{)) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }"","msg"v":":0,"name":"crucible"Crucible stats registered with UUID: 0c15e6ed-424b-4937-a4ba-572efa006555,""level":50,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747239949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":","downstairs"time":"}
2023-09-22T23:15:22.747246131Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":"4759msg":"}
{"[1] Reports error GenericError(msg"\":"bad\"Crucible 0c15e6ed-424b-4937-a4ba-572efa006555 has session id: 1894bd35-ed67-4d23-a2e0-dfa3df762dc8","v":0,"name":"crucible","level":30) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:22.747325216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:22.747339485Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"[0] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) New New New ds_transition to WaitActive"},"
v":0,"name":"crucible"{,"level":30"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747379555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"time":"}
2023-09-22T23:15:22.747389657Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid"":msg4759":","":"downstairs"[0] Transition from New to WaitActive"}
,"v":0,"name":"{crucible","level":"30msg":"[1] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747431858Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:22.747439911Z",""hostnamemsg""::""ip-10-150-1-55.us-west-2.compute.internal","pid":4759,[0] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) WaitActive New New ds_transition to WaitQuorum""":","downstairs"v":}0
,"name":"crucible","level":{30"msg":"[1] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":,30"time":"2023-09-22T23:15:22.747482859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"{time":"2023-09-22T23:15:22.747497786Z""msg":","hostname":"[0] Transition from WaitActive to WaitQuorum","v"ip-10-150-1-55.us-west-2.compute.internal":,"0pid":,4759"name":"}crucible"
,"level":30{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747534802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"{2023-09-22T23:15:22.747547205Z"",msg":""hostname":"ip-10-150-1-55.us-west-2.compute.internal[0] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) WaitQuorum New New ds_transition to Active"",",v":"0pid",":name":"4759crucible","level":}30
{,"time":"2023-09-22T23:15:22.74758087Z",""hostname":"msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
RE:0 Wait for result from reopen command 1003:4"{,"v""msg":":0[0] Transition from WaitQuorum to Active",","name":v":"0,crucible""name,":""crucible"level":,"30level":30,"time":"2023-09-22T23:15:22.747614884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":""msg":"2023-09-22T23:15:22.747614634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",[1] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) Active New New ds_transition to WaitActive"",pid":"4759v":0,"}name":"
crucible","level":30{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible",",time":""level":502023-09-22T23:15:22.74764478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":""msg":"2023-09-22T23:15:22.747654796Z","[1] Transition from New to WaitActivehostname":"","v":0ip-10-150-1-55.us-west-2.compute.internal",",name":""crucible"pid":,"4759level"}:30
{"msg":"[2] client skip 4 in process jobs because fault","v":0,","timename":"":"crucible","level2023-09-22T23:15:22.747676752Z"":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{,"time":""msg":"2023-09-22T23:15:22.747691485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759[1] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) Active WaitActive New ds_transition to WaitQuorum",,""v":0",":name":""crucible"downstairs,""level":}30
,"time":"{2023-09-22T23:15:22.747713976Z",""hostnamemsg":"":"[2] changed 0 jobs to fault skipped","ip-10-150-1-55.us-west-2.compute.internal"v":,"0pid":,"4759name":}"
crucible","{level":30"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747741276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",","time":"pid":47592023-09-22T23:15:22.747746116Z,""",":"hostname":downstairs""}
ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{{"msg":""msg":"[2] 1be45e4a-5a68-4e53-8c36-ebf527273e3d (1cc0e6a6-7f15-4d27-989e-892a7dc5f1be) Active Faulted LiveRepair ds_transition to Faulted"[1] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) Active WaitQuorum New ds_transition to Active",,""v":0v":0,","name":"name":"crucible"crucible",","levellevel":"30:30,"time":","time2023-09-22T23:15:22.747789594Z"",":"hostname":"2023-09-22T23:15:22.747790049Z","ip-10-150-1-55.us-west-2.compute.internal","hostname":"pid":4759}
ip-10-150-1-55.us-west-2.compute.internal","pid{":4759"}msg":"
[1] Transition from WaitQuorum to Active","v{":0,""msgname"":":"crucible","level":30[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747820088Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal",":"pid":47592023-09-22T23:15:22.747823299Z}
","hostname":"{"ip-10-150-1-55.us-west-2.compute.internal"msg":","pid":4759}
[2] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) Active Active New ds_transition to WaitActive","v":0{,"name":""crucible"msg,"":"level":30RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:22.747852798Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid2023-09-22T23:15:22.747855979Z"",":4759hostname":}"
ip-10-150-1-55.us-west-2.compute.internal","pid":{4759}
"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747879635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747900802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.7479191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747940411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.747973613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"0c15e6ed-424b-4937-a4ba-572efa006555 is now active with session: f68cb12b-8c72-4782-bbfd-626dd395c9fb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748011647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748049237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748084853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748123179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748163326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748204455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"{2023-09-22T23:15:22.748244004Z","hostname":""msg"ip-10-150-1-55.us-west-2.compute.internal":","pid":4759Crucible stats registered with UUID: ac3b0226-dead-4e0a-a77b-1f009b390e9a"}
,"v":0,"name":"{crucible","level":30"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748276424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4759"}time"
:"2023-09-22T23:15:22.748284167Z{",""hostname":msg":""ip-10-150-1-55.us-west-2.compute.internal","Crucible ac3b0226-dead-4e0a-a77b-1f009b390e9a has session id: 80f29230-7c3e-4c9e-8ba7-584487f3d649"pid",":v4759":0,"}name":
"crucible","level":30{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]",,""vtime":":"0,"2023-09-22T23:15:22.748325954Z"name",":hostname":""crucible"ip-10-150-1-55.us-west-2.compute.internal",,""pid":level4759":}30
{"msg":"[0] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748348948Z","hostname":","time":"2023-09-22T23:15:22.748362814Z"ip-10-150-1-55.us-west-2.compute.internal",",hostname":""pid":ip-10-150-1-55.us-west-2.compute.internal4759","pid":}4759
}
{{""msgmsg":"":"[0] Transition from New to WaitActive","v":0,"RE:0 close id:1000 queued, notify DSname":""crucible",,""level":v"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748399187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","[0] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) WaitActive New New ds_transition to WaitQuorum"time,""v":0:,""name":"crucible","2023-09-22T23:15:22.748404452Zlevel":"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"}time":
"2023-09-22T23:15:22.748426203Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4759"}msg"
:"{"msg":"RE:0 Wait for result from close command 1000:1","[0] Transition from WaitActive to WaitQuorum"v",:"0v",:"0,name"":name":""crucible","crucible"level":,30"level":30,"time":"2023-09-22T23:15:22.748464937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":","time":"[0] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) WaitQuorum New New ds_transition to Active","v":0,"2023-09-22T23:15:22.748466785Z"name":","crucible","hostname":level":"30ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
,"time":"2023-09-22T23:15:22.748491608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748513661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748534065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748552127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748572769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748598345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.74862025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748638759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748659094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.74867717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748697792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748716143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748736513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748754679Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"ac3b0226-dead-4e0a-a77b-1f009b390e9a is now active with session: 733e1a80-c7f6-4e41-9e2f-675bfc296009","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748774629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748795019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748813176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748833768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748852244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748872989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748891605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748909331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748932594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748951732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.748970119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.873298776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.873334052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.873353837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.87337196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:22.873390598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.873409201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:22.8734273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:22.873445516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 41428a10-9420-4a67-b738-1e9612812de2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.873871387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 41428a10-9420-4a67-b738-1e9612812de2 has session id: 75e0b9c2-4a79-4bee-bb52-2513fd0a37bd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.873905025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.873927149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.873946024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.873966547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.873984892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874005393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874023436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874043459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874068303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874090234Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.87410872Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874129109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874147045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874167282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874185339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874205699Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874223846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874244129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874262069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"41428a10-9420-4a67-b738-1e9612812de2 is now active with session: bb21993f-9652-4c5a-822b-f68bed4ca967","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.87428178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.87430203Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874319888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874340341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.87435857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874379152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874402243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874421424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874440207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874458152Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:22.874476426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.736753397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.736812317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:23.736831542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 9a789d93-ee2b-4070-ad87-88d89554e7ff","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737390709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 9a789d93-ee2b-4070-ad87-88d89554e7ff has session id: 1d38b817-30d0-4b12-9e45-a0a733ff22aa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737424326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737446522Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737465437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.73748635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737505329Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737525684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737544259Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737564521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.73758249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737602912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737631789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737653791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.73767197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737692173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737709994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737730415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737748864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737769342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737787408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"9a789d93-ee2b-4070-ad87-88d89554e7ff is now active with session: 059a98cb-d0c4-4eed-b294-64d16509a464","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737807007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737827162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737845033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active Faulted Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737865351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737883714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active LiveRepairReady Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737904266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.73792263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737940578Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737963732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.737983069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.738001512Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:23.748014217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1000, DownstairsIO { ds_id: JobId(1000), guest_id: 1, work: ExtentFlushClose { dependencies: [], extent: 0, flush_number: 0, gen_number: 0, source_downstairs: ClientId(0), repair_downstairs: [ClientId(2)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:23.748062508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 2 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.748083816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.748103149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 0c15e6ed-424b-4937-a4ba-572efa006555 (f68cb12b-8c72-4782-bbfd-626dd395c9fb) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.748125042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.748143614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.74816142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.748179692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1000 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:23.748197606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.748215856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.750042408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.750079267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.750101457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.874403791Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.87444737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.87446696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:23.874487638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.738979657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.739032605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.73905403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.749437399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:24.749472162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.749491633Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.750389826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.75042655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:24.875817103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:24.875880336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.87590178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.875921674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.875943538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.875971339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.875990757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:24.876009195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.876027324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.876046466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 41428a10-9420-4a67-b738-1e9612812de2 (bb21993f-9652-4c5a-822b-f68bed4ca967) Active Faulted LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.876068013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.876086135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:24.876104434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.739346055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.739399778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.749757337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.749791048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.749810381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.749828576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:25.749846939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.749865645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:25.749883878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:25.749901819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
----------------------------------------------------------------
 Crucible gen:0 GIO:true work queues:  Upstairs:0  downstairs:4
GW_ID      ACK  DSID    TYPE  BLOCKS  DS:0  DS:1  DS:2  REPLAY
    1    Acked  1000  FClose       0  Done  Done   Err  false
    2    Acked  1001    NoOp       0  Done  Done  Skip  false
    3    Acked  1002    NoOp       0  Done  Done  Skip  false
    4    Acked  1003  Reopen       0  Done  Done  Skip  false
   STATES      DS:0   DS:1   DS:2   TOTAL
    New           0      0      0      0
    Sent          0      0      0      0
    Done          4      4      0      8
    Skipped       0      0      3      3
    Error         0      0      1      1
Last Flush: 0 0 0 
Downstairs last five completed:
Upstairs last five completed:      4    3    2    1
test live_repair::repair_test::test_repair_extent_close_fails_all ... ok
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.751673092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:25.751760938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1003, DownstairsIO { ds_id: JobId(1003), guest_id: 4, work: ExtentLiveReopen { dependencies: [JobId(1000), JobId(1001), JobId(1002)], extent: 0 }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:25.751797654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.75181821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 0 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.751838097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] ac3b0226-dead-4e0a-a77b-1f009b390e9a (733e1a80-c7f6-4e41-9e2f-675bfc296009) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.751860363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.751878874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.751897355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:25.751915619Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:25.7519335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_extent_fail_reopen_all ... ok
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.876232342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.876286035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.876305985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.876324188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:25.87634271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.876372664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:25.876392523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:25.876410552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 77bc3a9f-f333-44eb-ad87-4ee9f025b577","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.876947067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 77bc3a9f-f333-44eb-ad87-4ee9f025b577 has session id: aaeb38c7-4ff1-4b76-b886-a1af536d6ef6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.876981657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.87700446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877023978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877044788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877063416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877083816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.8771022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877122772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877141007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877161558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877179865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877200355Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877218519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.87723898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877263378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877285277Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877303697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877324105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.87734219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"77bc3a9f-f333-44eb-ad87-4ee9f025b577 is now active with session: 579d228d-6578-4a36-9875-7a6297b92e8b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877362021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877382208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877400302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877420978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877439228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.87745978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877478206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877496181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877514725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877532766Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:25.877550999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.740670758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:26.7407484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:26.740797826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.74082041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.740840388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active LiveRepair Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.740862306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.740880793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:26.74089909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.740917845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.74093702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 9a789d93-ee2b-4070-ad87-88d89554e7ff (059a98cb-d0c4-4eed-b294-64d16509a464) Active LiveRepair Faulted ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.740958825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.740977065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.740995582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the close job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.878600033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.878650448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.878669496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:26.878690477Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:27.880016545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1001, DownstairsIO { ds_id: JobId(1001), guest_id: 2, work: ExtentLiveNoOp { dependencies: [JobId(1000)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:27.880092454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:27.880114725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:27.880134357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 77bc3a9f-f333-44eb-ad87-4ee9f025b577 (579d228d-6578-4a36-9875-7a6297b92e8b) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:27.880156421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:27.880174787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:27.880193028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1001 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:27.880211428Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:27.880230015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.582989764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.584574869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.741149808Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.741190325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:28.741209367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"err:2 or:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.741369944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 46ceccb1-8e43-4e30-90b6-a8d90c7cea97","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.741817958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 has session id: de9240d0-ab6d-4567-a2ad-e7a003b12882","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.741852154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.74187468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.741904797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.741927841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.741946789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.741967508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.741986071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742006564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742024968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742045574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742064004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742084338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.74210267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742123081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742141357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742161964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742180594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742201154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742219413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"46ceccb1-8e43-4e30-90b6-a8d90c7cea97 is now active with session: c4c9ffcf-62b7-4d90-8f2e-3be934889a7f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742239372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742264731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742284198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742305139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.74232374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742344667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742363359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.74238173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742400547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742418986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.742437549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.881390608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the NoOp job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.881437322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.881456526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the Reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.881474542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:28.881492975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:28.881511521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1003 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:28.881529802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:28.881547485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_extent_repair_fails_all ... ok
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:29.743420853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:29.743482647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:29.743505506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:30.744793659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:30.744848593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:31.745213857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:31.745281757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([InProgress, New, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:31.745315198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:31.745335884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:31.745355518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[0] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:31.745377682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:31.745396252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:31.745414682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:31.745433259Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:31.745452604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 46ceccb1-8e43-4e30-90b6-a8d90c7cea97 (c4c9ffcf-62b7-4d90-8f2e-3be934889a7f) Faulted Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:31.745474436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:31.74549286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:31.74552325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.746649146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.746701602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:33.746720329Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"err:0 or:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.746861069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: 50bf3ac1-5436-4030-be9f-16763e0c8bad","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747474542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible 50bf3ac1-5436-4030-be9f-16763e0c8bad has session id: 9b00af65-f93d-4019-ab40-d9c93fcb3b99","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747509506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747532063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747550904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747571931Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747590462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747610778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747629059Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747649359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747667411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747687916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747706143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747726413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747744536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747773907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747793652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747814418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747832782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747853353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747871653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"50bf3ac1-5436-4030-be9f-16763e0c8bad is now active with session: 3b5addfa-5709-424a-bddc-395f274761b4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747891521Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747911927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747930045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747950716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.74796906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.747989916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.748008313Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.748026408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.748044967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.748063154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:33.748081632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:34.748078022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:34.748140416Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:34.748162885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:35.749564428Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:35.7496215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:36.7508642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:36.750931547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, InProgress, New]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:36.750964624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:36.750985465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:36.751005393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[1] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:36.751027341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:36.751045721Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:36.751064013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:36.7510825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:36.751102027Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] 50bf3ac1-5436-4030-be9f-16763e0c8bad (3b5addfa-5709-424a-bddc-395f274761b4) Active Faulted LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:36.751123834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:36.751142274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:36.7511713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.585560276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.587717668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.752290778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.752341733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:38.752361012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"err:1 or:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.752504461Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible stats registered with UUID: ac0a948c-9d98-4276-8284-4295e9402293","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.752950312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Crucible ac0a948c-9d98-4276-8284-4295e9402293 has session id: 0f185046-3285-4801-a97b-f4ca49d7f69a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.752984268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753006837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753025917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) WaitActive New New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753046623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753065165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) WaitQuorum New New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753085508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753103645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) Active New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753123723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753141647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) Active WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753162149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753180372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) Active WaitQuorum New ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753211991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[1] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753231532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) Active Active New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753251996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753270115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) Active Active WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753290582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753308803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) Active Active WaitQuorum ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753329149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753347009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"ac0a948c-9d98-4276-8284-4295e9402293 is now active with session: bb779cc3-1172-4b3f-b5ea-1d842045ac13","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753366639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753386643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753404608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) Active Active Faulted ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753425014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753443426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) Active Active LiveRepairReady ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.75346388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753483065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for Close + ReOpen jobs","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753501439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 repair extent with ids 1000,1001,1002,1003 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753520081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 close id:1000 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.753538037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from close command 1000:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:38.75356135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 3 jobs (currently 2)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:39.754659563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"No repair needed for extent 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:39.754708227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"RE:0 Wait for result from repair command 1001:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:39.754729447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Waiting for 4 jobs (currently 3)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:40.755069557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from NoOp command 1002:3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:40.755128506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now move the NoOp job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:41.756402197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] DS Reports error Err(GenericError(\"bad\")) on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:41.756494269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] Reports error GenericError(\"bad\") on job 1002, DownstairsIO { ds_id: JobId(1002), guest_id: 3, work: ExtentLiveNoOp { dependencies: [JobId(1000), JobId(1001)] }, state: ClientData([Done, Done, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 2 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:41.756531198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] client skip 4 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:41.756552628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:41.756572293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs"}
{"msg":"[2] ac0a948c-9d98-4276-8284-4295e9402293 (bb779cc3-1172-4b3f-b5ea-1d842045ac13) Active Active LiveRepair ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:41.756594319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[2] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:41.756613051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Extent 0 close id:1002 Failed: Error: bad","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:15:41.756631323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Wait for result from reopen command 1003:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:41.756649943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Finally, move the ReOpen job forward","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:43.757783458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"Now ACK the reopen job","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:43.757836969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:15:43.75786828Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
test live_repair::repair_test::test_repair_extent_fail_noop_all ... ok
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:48.586665631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:48.588919616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
test dummy_downstairs_tests::protocol_test::test_error_during_live_repair_no_halt has been running for over 60 seconds
test dummy_downstairs_tests::protocol_test::test_successful_live_repair has been running for over 60 seconds
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:58.588254214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:15:58.594049946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:08.59341722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:08.599187175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:18.595496787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:18.605275368Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:28.598947676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:28.608955885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:38.600736422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:38.613887988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:48.606461647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:48.614866137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:58.610302396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:16:58.621927606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:08.615086042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:08.62592421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:18.620908559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:18.631744587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:28.629216469Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:28.637806232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:38.63786222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:38.640862681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:48.642712939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:48.644401109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:58.64840206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:17:58.654171015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:08.653324536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:08.656578093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:18.662424609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:18.663874678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:28.664935704Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:28.666572567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:38.668645334Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:38.675303171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:48.670443959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:48.676770971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:58.678269449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:18:58.687531459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:08.689413576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:08.699362093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:18.696245998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:18.701944636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:28.707480595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:28.70834726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:38.718928289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:38.719510011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:48.723987985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:48.725970274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:58.72847769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:19:58.733770606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:08.739744586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:08.74233362Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:18.743815862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"{downstairs":1}
"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:18.743880475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:28.746677962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:28.747650214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:38.757544627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:38.758614637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:48.763071264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:48.769263948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:58.766566154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:20:58.774755473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:08.770158602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:08.778194713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[up] downstairs 0 failed, too many outstanding jobs 57001","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:17.262069086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] client skip 57001 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:17.262132853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] changed 57001 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:17.331777043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 85682a66-27fd-403c-bf22-d04b4ff66805 (5d828379-bf79-455f-9270-e82849f9ecba) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:17.420106971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:17.420158529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:17.640850425Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:18.782120498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] will exit pm_task, this downstairs Faulted","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:18.782367482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"127.0.0.1:39040: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Faulted)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:18.782505803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] 85682a66-27fd-403c-bf22-d04b4ff66805 Gone missing, transition from Faulted to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:18.782535579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 85682a66-27fd-403c-bf22-d04b4ff66805 connection to 127.0.0.1:39040 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:18.782557083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] 127.0.0.1:39040 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:18.782578317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"85682a66-27fd-403c-bf22-d04b4ff66805 Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:18.782598481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 127.0.0.1:39040 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:18.782618083Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"spawn_message_receiver saw disconnect, bailing","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:18.782773597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:18.791394785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] 85682a66-27fd-403c-bf22-d04b4ff66805 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:19.793889254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:39040 in state Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:19.793957537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[up] downstairs 0 failed, too many outstanding jobs 57001","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:19.805276904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] client skip 57001 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:19.805312098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] changed 57001 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:19.873818064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) Active Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:19.961155326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from Active to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:19.961210093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] flow control end ","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:20.465365711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"could not send read response for job 1 = 1001: Broken pipe (os error 32)","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:29.483974801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 85682a66-27fd-403c-bf22-d04b4ff66805, session_id: 5d828379-bf79-455f-9270-e82849f9ecba, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484152322Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484183447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] upstairs guest_io_ready=TRUE, promote! session 5d828379-bf79-455f-9270-e82849f9ecba","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484277473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: 85682a66-27fd-403c-bf22-d04b4ff66805, session_id: 5d828379-bf79-455f-9270-e82849f9ecba, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484386351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484528777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] downstairs client at 127.0.0.1:39040 has UUID b0da680b-dcef-4405-a0bc-9a419607b615","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484633538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: b0da680b-dcef-4405-a0bc-9a419607b615, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.48466588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Returning client:0 UUID:b0da680b-dcef-4405-a0bc-9a419607b615 matches","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.48468772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"85682a66-27fd-403c-bf22-d04b4ff66805 Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484717646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484739292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] 85682a66-27fd-403c-bf22-d04b4ff66805 (5d828379-bf79-455f-9270-e82849f9ecba) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484893658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484921388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] new RM replaced this: Some(RegionMetadata { generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dirty: [false, false, false, false, false, false, false, false, false, false] })","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:29.484946312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 85682a66-27fd-403c-bf22-d04b4ff66805 Enter Ready for LiveRepair mode","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:29.484967053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.484986067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 127.0.0.1:39040 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.485511664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"85682a66-27fd-403c-bf22-d04b4ff66805 LiveRepairReady Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.485539346Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:29.485558949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.231601781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":2}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.231661073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":3}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.487104573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 85682a66-27fd-403c-bf22-d04b4ff66805 (5d828379-bf79-455f-9270-e82849f9ecba) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.487155676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.48717665Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Live Repair started","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.487195747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Live Repair main task begins.","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.487215111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"Start Live Repair of extents 0 to 10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.487236548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"Start extent 0 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.487268562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:0 repair extent with ids 58201,58202,58203,58204 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.487291374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:0 close id:58201 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.487311279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:0 Wait for result from close command 58201:57202","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.487331155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58201 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.48737065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58204 final dependency list [JobId(58201), JobId(58202), JobId(58203)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.487475866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58205 final dependency list [JobId(58204), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.488094724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 0:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.488548687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58206 final dependency list [JobId(58205), JobId(58204), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.488743016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 0 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.497395912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:0 Wait for result from repair command 58202:57203","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.497427487Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58202 final dependency list [JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.497450849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:0 Wait for result from NoOp command 58203:57204","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.498023594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58203 final dependency list [JobId(58201), JobId(58202)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.498053209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:0 Wait for result from reopen command 58204:57205","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.500785134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Start extent 1 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.500812109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:1 repair extent with ids 58225,58226,58227,58228 deps:[JobId(58208), JobId(58207)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.500834933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:1 close id:58225 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.500854597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:1 Wait for result from close command 58225:57226","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.500874466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58225 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.501002397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58228 final dependency list [JobId(58225), JobId(58226), JobId(58227)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.501032935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58229 final dependency list [JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.501562909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58230 final dependency list [JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.502135074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58231 final dependency list [JobId(58228), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.502705603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 1:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.50312223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58232 final dependency list [JobId(58231), JobId(58228), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.503294298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 1 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.511079879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:1 Wait for result from repair command 58226:57227","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.511108916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58226 final dependency list [JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.511135596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:1 Wait for result from NoOp command 58227:57228","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.511674654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58227 final dependency list [JobId(58225), JobId(58226)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.51170433Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:1 Wait for result from reopen command 58228:57229","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.51443673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Start extent 2 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.514463688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:2 repair extent with ids 58249,58250,58251,58252 deps:[JobId(58234), JobId(58233), JobId(58210), JobId(58209)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.514487348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:2 close id:58249 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.514507243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:2 Wait for result from close command 58249:57250","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.514527295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58249 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.514657178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58252 final dependency list [JobId(58249), JobId(58250), JobId(58251)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.5146878Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58253 final dependency list [JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.515256889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58254 final dependency list [JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.515832035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58255 final dependency list [JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.516395764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58256 final dependency list [JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.516964092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58257 final dependency list [JobId(58252), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.517538405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 2:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.517991381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58258 final dependency list [JobId(58257), JobId(58252), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.518166283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 2 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.525197606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:2 Wait for result from repair command 58250:57251","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.525226645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58250 final dependency list [JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.525254814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:2 Wait for result from NoOp command 58251:57252","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.525798013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58251 final dependency list [JobId(58249), JobId(58250)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.52582946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:2 Wait for result from reopen command 58252:57253","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.528619112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Start extent 3 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.528645933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:3 repair extent with ids 58273,58274,58275,58276 deps:[JobId(58260), JobId(58259), JobId(58236), JobId(58235), JobId(58212), JobId(58211)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.528670143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:3 close id:58273 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.528697646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:3 Wait for result from close command 58273:57274","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.528719327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58273 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.528849917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58276 final dependency list [JobId(58273), JobId(58274), JobId(58275)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.528879329Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58277 final dependency list [JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.529432935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58278 final dependency list [JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.530016986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58279 final dependency list [JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.530594715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58280 final dependency list [JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.531172653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58281 final dependency list [JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.531750278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58282 final dependency list [JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.532336922Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58283 final dependency list [JobId(58276), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.532921943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 3:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.533355994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58284 final dependency list [JobId(58283), JobId(58276), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.533531256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 3 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.539772528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:3 Wait for result from repair command 58274:57275","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.539801436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58274 final dependency list [JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.53983179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:3 Wait for result from NoOp command 58275:57276","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.54038069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58275 final dependency list [JobId(58273), JobId(58274)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.540412492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:3 Wait for result from reopen command 58276:57277","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.543310927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Start extent 4 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.543337409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:4 repair extent with ids 58297,58298,58299,58300 deps:[JobId(58286), JobId(58285), JobId(58262), JobId(58261), JobId(58238), JobId(58237), JobId(58214), JobId(58213)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.543362523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:4 close id:58297 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.543416142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:4 Wait for result from close command 58297:57298","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.543442002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58297 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.54356207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58300 final dependency list [JobId(58297), JobId(58298), JobId(58299)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.543590903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58301 final dependency list [JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.544154585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58302 final dependency list [JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.544747257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58303 final dependency list [JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.545332733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58304 final dependency list [JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.545921216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58305 final dependency list [JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.546504925Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58306 final dependency list [JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.547094927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58307 final dependency list [JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.547685381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58308 final dependency list [JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.548289869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58309 final dependency list [JobId(58300), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.54886905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 4:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.54931173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58310 final dependency list [JobId(58309), JobId(58300), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.549489359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 4 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.554899393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:4 Wait for result from repair command 58298:57299","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.554928043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58298 final dependency list [JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.554958373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:4 Wait for result from NoOp command 58299:57300","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.555508644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58299 final dependency list [JobId(58297), JobId(58298)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.555542333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:4 Wait for result from reopen command 58300:57301","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.558494697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Start extent 5 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.55852113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:5 repair extent with ids 58321,58322,58323,58324 deps:[JobId(58312), JobId(58311), JobId(58288), JobId(58287), JobId(58264), JobId(58263), JobId(58240), JobId(58239), JobId(58216), JobId(58215)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.558548079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:5 close id:58321 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.558581427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:5 Wait for result from close command 58321:57322","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.558602447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58321 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.558737308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58324 final dependency list [JobId(58321), JobId(58322), JobId(58323)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.558771678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58325 final dependency list [JobId(58302), JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.559344244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58326 final dependency list [JobId(58325), JobId(58302), JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.559939386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58327 final dependency list [JobId(58304), JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.560537282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58328 final dependency list [JobId(58327), JobId(58304), JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.561132337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58329 final dependency list [JobId(58306), JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.561761296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58330 final dependency list [JobId(58329), JobId(58306), JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.562364996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58331 final dependency list [JobId(58308), JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.562959774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58332 final dependency list [JobId(58331), JobId(58308), JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.56355588Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58333 final dependency list [JobId(58310), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.564150112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58334 final dependency list [JobId(58333), JobId(58310), JobId(58309), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.56474524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58335 final dependency list [JobId(58324), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.565334439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 5:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.5657799Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58336 final dependency list [JobId(58335), JobId(58324), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.565963683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 5 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.570437801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:5 Wait for result from repair command 58322:57323","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.570466331Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58322 final dependency list [JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.570496928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:5 Wait for result from NoOp command 58323:57324","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.571057428Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58323 final dependency list [JobId(58321), JobId(58322)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.571091888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:5 Wait for result from reopen command 58324:57325","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.574171337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Start extent 6 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.57419972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:6 repair extent with ids 58345,58346,58347,58348 deps:[JobId(58338), JobId(58337), JobId(58314), JobId(58313), JobId(58290), JobId(58289), JobId(58266), JobId(58265), JobId(58242), JobId(58241), JobId(58218), JobId(58217)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.574232691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:6 close id:58345 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.574265387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:6 Wait for result from close command 58345:57346","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.574286309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58345 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.574424762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58348 final dependency list [JobId(58345), JobId(58346), JobId(58347)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.574454978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58349 final dependency list [JobId(58326), JobId(58302), JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.575072918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58350 final dependency list [JobId(58349), JobId(58326), JobId(58325), JobId(58302), JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.575686303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58351 final dependency list [JobId(58328), JobId(58304), JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.5763185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58352 final dependency list [JobId(58351), JobId(58328), JobId(58327), JobId(58304), JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.576936036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58353 final dependency list [JobId(58330), JobId(58306), JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.577553211Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58354 final dependency list [JobId(58353), JobId(58330), JobId(58329), JobId(58306), JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.578159965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58355 final dependency list [JobId(58332), JobId(58308), JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.578763201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58356 final dependency list [JobId(58355), JobId(58332), JobId(58331), JobId(58308), JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.579366151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58357 final dependency list [JobId(58334), JobId(58310), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.579969576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58358 final dependency list [JobId(58357), JobId(58334), JobId(58333), JobId(58310), JobId(58309), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.580581693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58359 final dependency list [JobId(58336), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.581215496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58360 final dependency list [JobId(58359), JobId(58336), JobId(58335), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.581819511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58361 final dependency list [JobId(58348), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.582423551Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 6:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.582878222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58362 final dependency list [JobId(58361), JobId(58348), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.583057024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 6 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.586662721Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:6 Wait for result from repair command 58346:57347","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.586700307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58346 final dependency list [JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.586723768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:6 Wait for result from NoOp command 58347:57348","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.587286012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58347 final dependency list [JobId(58345), JobId(58346)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.587320263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:6 Wait for result from reopen command 58348:57349","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.590441596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Start extent 7 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.590468433Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:7 repair extent with ids 58369,58370,58371,58372 deps:[JobId(58364), JobId(58363), JobId(58340), JobId(58339), JobId(58316), JobId(58315), JobId(58292), JobId(58291), JobId(58268), JobId(58267), JobId(58244), JobId(58243), JobId(58220), JobId(58219)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.590512906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:7 close id:58369 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.590546079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:7 Wait for result from close command 58369:57370","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.590567228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58369 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.590708031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58372 final dependency list [JobId(58369), JobId(58370), JobId(58371)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.590738489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58373 final dependency list [JobId(58350), JobId(58326), JobId(58302), JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.59134082Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58374 final dependency list [JobId(58373), JobId(58350), JobId(58349), JobId(58326), JobId(58325), JobId(58302), JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.591958947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58375 final dependency list [JobId(58352), JobId(58328), JobId(58304), JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.592583363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58376 final dependency list [JobId(58375), JobId(58352), JobId(58351), JobId(58328), JobId(58327), JobId(58304), JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.593198407Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58377 final dependency list [JobId(58354), JobId(58330), JobId(58306), JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.593821761Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58378 final dependency list [JobId(58377), JobId(58354), JobId(58353), JobId(58330), JobId(58329), JobId(58306), JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.594433998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58379 final dependency list [JobId(58356), JobId(58332), JobId(58308), JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.595071447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58380 final dependency list [JobId(58379), JobId(58356), JobId(58355), JobId(58332), JobId(58331), JobId(58308), JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.595685016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58381 final dependency list [JobId(58358), JobId(58334), JobId(58310), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.596298068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58382 final dependency list [JobId(58381), JobId(58358), JobId(58357), JobId(58334), JobId(58333), JobId(58310), JobId(58309), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.596914257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58383 final dependency list [JobId(58360), JobId(58336), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.597527587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58384 final dependency list [JobId(58383), JobId(58360), JobId(58359), JobId(58336), JobId(58335), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.598142827Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58385 final dependency list [JobId(58362), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.598758542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58386 final dependency list [JobId(58385), JobId(58362), JobId(58361), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.599366365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58387 final dependency list [JobId(58372), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.600002288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 7:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.600462005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58388 final dependency list [JobId(58387), JobId(58372), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.600647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 7 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.603307171Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:7 Wait for result from repair command 58370:57371","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.603337278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58370 final dependency list [JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.603367202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:7 Wait for result from NoOp command 58371:57372","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.603925156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58371 final dependency list [JobId(58369), JobId(58370)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.603960163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:7 Wait for result from reopen command 58372:57373","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.607153696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Start extent 8 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.607180246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:8 repair extent with ids 58393,58394,58395,58396 deps:[JobId(58390), JobId(58389), JobId(58366), JobId(58365), JobId(58342), JobId(58341), JobId(58318), JobId(58317), JobId(58294), JobId(58293), JobId(58270), JobId(58269), JobId(58246), JobId(58245), JobId(58222), JobId(58221)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.607268336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:8 close id:58393 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.607295282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:8 Wait for result from close command 58393:57394","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.607315962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58393 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.60744641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58396 final dependency list [JobId(58393), JobId(58394), JobId(58395)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.607476728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58397 final dependency list [JobId(58374), JobId(58350), JobId(58326), JobId(58302), JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.608091797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58398 final dependency list [JobId(58397), JobId(58374), JobId(58373), JobId(58350), JobId(58349), JobId(58326), JobId(58325), JobId(58302), JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.608724874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58399 final dependency list [JobId(58376), JobId(58352), JobId(58328), JobId(58304), JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.609390508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58400 final dependency list [JobId(58399), JobId(58376), JobId(58375), JobId(58352), JobId(58351), JobId(58328), JobId(58327), JobId(58304), JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.610017756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58401 final dependency list [JobId(58378), JobId(58354), JobId(58330), JobId(58306), JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.610638693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58402 final dependency list [JobId(58401), JobId(58378), JobId(58377), JobId(58354), JobId(58353), JobId(58330), JobId(58329), JobId(58306), JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.611260546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58403 final dependency list [JobId(58380), JobId(58356), JobId(58332), JobId(58308), JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.611881924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58404 final dependency list [JobId(58403), JobId(58380), JobId(58379), JobId(58356), JobId(58355), JobId(58332), JobId(58331), JobId(58308), JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.612503518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58405 final dependency list [JobId(58382), JobId(58358), JobId(58334), JobId(58310), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.613134052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58406 final dependency list [JobId(58405), JobId(58382), JobId(58381), JobId(58358), JobId(58357), JobId(58334), JobId(58333), JobId(58310), JobId(58309), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.613767868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58407 final dependency list [JobId(58384), JobId(58360), JobId(58336), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.614411705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58408 final dependency list [JobId(58407), JobId(58384), JobId(58383), JobId(58360), JobId(58359), JobId(58336), JobId(58335), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.61503192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58409 final dependency list [JobId(58386), JobId(58362), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.615653972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58410 final dependency list [JobId(58409), JobId(58386), JobId(58385), JobId(58362), JobId(58361), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.616272517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58411 final dependency list [JobId(58388), JobId(58372), JobId(58371), JobId(58370), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.61689338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58412 final dependency list [JobId(58411), JobId(58388), JobId(58387), JobId(58372), JobId(58371), JobId(58370), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.617513188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58413 final dependency list [JobId(58396), JobId(58393)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.618130196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 8:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.618613455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58414 final dependency list [JobId(58413), JobId(58396), JobId(58393)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.618793742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 8 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.620486048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:8 Wait for result from repair command 58394:57395","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.620514969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58394 final dependency list [JobId(58393)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.62055065Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:8 Wait for result from NoOp command 58395:57396","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.621110147Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58395 final dependency list [JobId(58393), JobId(58394)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.621146661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:8 Wait for result from reopen command 58396:57397","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.624432543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Start extent 9 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.6244599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:9 repair extent with ids 58417,58418,58419,58420 deps:[JobId(58416), JobId(58415), JobId(58392), JobId(58391), JobId(58368), JobId(58367), JobId(58344), JobId(58343), JobId(58320), JobId(58319), JobId(58296), JobId(58295), JobId(58272), JobId(58271), JobId(58248), JobId(58247), JobId(58224), JobId(58223)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.624549396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:9 close id:58417 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.624577481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:9 Wait for result from close command 58417:57418","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.624598102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58417 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.624729132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58420 final dependency list [JobId(58417), JobId(58418), JobId(58419)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.624763945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58421 final dependency list [JobId(58398), JobId(58374), JobId(58350), JobId(58326), JobId(58302), JobId(58278), JobId(58254), JobId(58230), JobId(58206), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.625401564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58422 final dependency list [JobId(58421), JobId(58398), JobId(58397), JobId(58374), JobId(58373), JobId(58350), JobId(58349), JobId(58326), JobId(58325), JobId(58302), JobId(58301), JobId(58278), JobId(58277), JobId(58254), JobId(58253), JobId(58230), JobId(58229), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.626042148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58423 final dependency list [JobId(58400), JobId(58376), JobId(58352), JobId(58328), JobId(58304), JobId(58280), JobId(58256), JobId(58232), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.626673004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58424 final dependency list [JobId(58423), JobId(58400), JobId(58399), JobId(58376), JobId(58375), JobId(58352), JobId(58351), JobId(58328), JobId(58327), JobId(58304), JobId(58303), JobId(58280), JobId(58279), JobId(58256), JobId(58255), JobId(58232), JobId(58231), JobId(58228), JobId(58227), JobId(58226), JobId(58225)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.627306918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58425 final dependency list [JobId(58402), JobId(58378), JobId(58354), JobId(58330), JobId(58306), JobId(58282), JobId(58258), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.627964428Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58426 final dependency list [JobId(58425), JobId(58402), JobId(58401), JobId(58378), JobId(58377), JobId(58354), JobId(58353), JobId(58330), JobId(58329), JobId(58306), JobId(58305), JobId(58282), JobId(58281), JobId(58258), JobId(58257), JobId(58252), JobId(58251), JobId(58250), JobId(58249)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.628597303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58427 final dependency list [JobId(58404), JobId(58380), JobId(58356), JobId(58332), JobId(58308), JobId(58284), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.629224839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58428 final dependency list [JobId(58427), JobId(58404), JobId(58403), JobId(58380), JobId(58379), JobId(58356), JobId(58355), JobId(58332), JobId(58331), JobId(58308), JobId(58307), JobId(58284), JobId(58283), JobId(58276), JobId(58275), JobId(58274), JobId(58273)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.629859804Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58429 final dependency list [JobId(58406), JobId(58382), JobId(58358), JobId(58334), JobId(58310), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.630484497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58430 final dependency list [JobId(58429), JobId(58406), JobId(58405), JobId(58382), JobId(58381), JobId(58358), JobId(58357), JobId(58334), JobId(58333), JobId(58310), JobId(58309), JobId(58300), JobId(58299), JobId(58298), JobId(58297)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.631113673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58431 final dependency list [JobId(58408), JobId(58384), JobId(58360), JobId(58336), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.631743523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58432 final dependency list [JobId(58431), JobId(58408), JobId(58407), JobId(58384), JobId(58383), JobId(58360), JobId(58359), JobId(58336), JobId(58335), JobId(58324), JobId(58323), JobId(58322), JobId(58321)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.632369087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58433 final dependency list [JobId(58410), JobId(58386), JobId(58362), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.633029889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58434 final dependency list [JobId(58433), JobId(58410), JobId(58409), JobId(58386), JobId(58385), JobId(58362), JobId(58361), JobId(58348), JobId(58347), JobId(58346), JobId(58345)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.633670372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58435 final dependency list [JobId(58412), JobId(58388), JobId(58372), JobId(58371), JobId(58370), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.634300722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58436 final dependency list [JobId(58435), JobId(58412), JobId(58411), JobId(58388), JobId(58387), JobId(58372), JobId(58371), JobId(58370), JobId(58369)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.634929825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58437 final dependency list [JobId(58414), JobId(58396), JobId(58395), JobId(58394), JobId(58393)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.635557274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58438 final dependency list [JobId(58437), JobId(58414), JobId(58413), JobId(58396), JobId(58395), JobId(58394), JobId(58393)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.636187639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58439 final dependency list [JobId(58420), JobId(58417)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.636827087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Write to Extent 9:0:9 under repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.63730288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58440 final dependency list [JobId(58439), JobId(58420), JobId(58417)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.63748686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 9 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.638177446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:9 Wait for result from repair command 58418:57419","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.63820651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58418 final dependency list [JobId(58417)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.638241892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:9 Wait for result from NoOp command 58419:57420","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.638811009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58419 final dependency list [JobId(58417), JobId(58418)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.63884805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:9 Wait for result from reopen command 58420:57421","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.642272462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"LiveRepair final flush submitted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.642382917Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58441 final dependency list [JobId(58440), JobId(58439), JobId(58438), JobId(58437), JobId(58436), JobId(58435), JobId(58434), JobId(58433), JobId(58432), JobId(58431), JobId(58430), JobId(58429), JobId(58428), JobId(58427), JobId(58426), JobId(58425), JobId(58424), JobId(58423), JobId(58422), JobId(58421), JobId(58420), JobId(58419), JobId(58418), JobId(58417), JobId(58414), JobId(58413), JobId(58412), JobId(58411), JobId(58410), JobId(58409), JobId(58408), JobId(58407), JobId(58406), JobId(58405), JobId(58404), JobId(58403), JobId(58402), JobId(58401), JobId(58400), JobId(58399), JobId(58398), JobId(58397), JobId(58396), JobId(58395), JobId(58394), JobId(58393), JobId(58388), JobId(58387), JobId(58386), JobId(58385), JobId(58384), JobId(58383), JobId(58382), JobId(58381), JobId(58380), JobId(58379), JobId(58378), JobId(58377), JobId(58376), JobId(58375), JobId(58374), JobId(58373), JobId(58372), JobId(58371), JobId(58370), JobId(58369), JobId(58362), JobId(58361), JobId(58360), JobId(58359), JobId(58358), JobId(58357), JobId(58356), JobId(58355), JobId(58354), JobId(58353), JobId(58352), JobId(58351), JobId(58350), JobId(58349), JobId(58348), JobId(58347), JobId(58346), JobId(58345), JobId(58336), JobId(58335), JobId(58334), JobId(58333), JobId(58332), JobId(58331), JobId(58330), JobId(58329), JobId(58328), JobId(58327), JobId(58326), JobId(58325), JobId(58324), JobId(58323), JobId(58322), JobId(58321), JobId(58310), JobId(58309), JobId(58308), JobId(58307), JobId(58306), JobId(58305), JobId(58304), JobId(58303), JobId(58302), JobId(58301), JobId(58300), JobId(58299), JobId(58298), JobId(58297), JobId(58284), JobId(58283), JobId(58282), JobId(58281), JobId(58280), JobId(58279), JobId(58278), JobId(58277), JobId(58276), JobId(58275), JobId(58274), JobId(58273), JobId(58258), JobId(58257), JobId(58256), JobId(58255), JobId(58254), JobId(58253), JobId(58252), JobId(58251), JobId(58250), JobId(58249), JobId(58232), JobId(58231), JobId(58230), JobId(58229), JobId(58228), JobId(58227), JobId(58226), JobId(58225), JobId(58206), JobId(58205), JobId(58204), JobId(58203), JobId(58202), JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.642707178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"LiveRepair final flush completed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.64391579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 85682a66-27fd-403c-bf22-d04b4ff66805 (5d828379-bf79-455f-9270-e82849f9ecba) LiveRepair Active Active ds_transition to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.643945236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from LiveRepair to Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:30.643965554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Live Repair returns Ok(())","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:30.643990968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
test dummy_downstairs_tests::protocol_test::test_successful_live_repair ... ok
{"msg":"sent read response for job 0 = 1000","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:31.926476285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"sent read response for job 1 = 1001","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:31.926648851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:31.926820423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] Dropping job 1000, this downstairs is faulted","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:31.926849982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"process_ds_operation error: No longer active","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:31.926871268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] will exit pm_task, this downstairs Faulted","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:31.926891327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"127.0.0.1:36984: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Faulted)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:31.927001179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa Gone missing, transition from Faulted to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:31.927031442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa connection to 127.0.0.1:36984 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:31.927053091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] 127.0.0.1:36984 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:31.927074526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"28a12560-c07c-41d3-8d35-7e06b0c337aa Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:31.92709476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 127.0.0.1:36984 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:31.927114586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"spawn_message_receiver saw disconnect, bailing","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:31.927134262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"sent read response for job 2 = 1002","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:31.927311768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"could not send read response for job 3 = 1003: Broken pipe (os error 32)","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:31.927485353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.672027435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":2}
{"msg":"responded to ping","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.672083026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":3}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.927764044Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:36984 in state Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.92783292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 28a12560-c07c-41d3-8d35-7e06b0c337aa, session_id: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.927880137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] upstairs guest_io_ready=TRUE, promote! session 7f1ba66d-4c7a-4444-a654-4b5bfde1244a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928009403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: 28a12560-c07c-41d3-8d35-7e06b0c337aa, session_id: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928101415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928242459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] downstairs client at 127.0.0.1:36984 has UUID 8dfe8526-b15f-436d-8c37-dad574df0099","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928348959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 8dfe8526-b15f-436d-8c37-dad574df0099, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928380915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Returning client:0 UUID:8dfe8526-b15f-436d-8c37-dad574df0099 matches","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928402007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"28a12560-c07c-41d3-8d35-7e06b0c337aa Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928421785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928441257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928636255Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928662906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] new RM replaced this: Some(RegionMetadata { generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dirty: [false, false, false, false, false, false, false, false, false, false] })","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:32.928687669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa Enter Ready for LiveRepair mode","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:32.928707963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928727038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 127.0.0.1:36984 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928746667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"28a12560-c07c-41d3-8d35-7e06b0c337aa LiveRepairReady Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928773004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:32.928793207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.930326954Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.930386308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.930407215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Live Repair started","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.930426064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Live Repair main task begins.","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:33.930445319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"Start Live Repair of extents 0 to 10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.930467213Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"Start extent 0 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.930487449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:0 repair extent with ids 58201,58202,58203,58204 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.930508509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:0 close id:58201 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.930527731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:0 Wait for result from close command 58201:57202","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.930547321Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58201 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.930606516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58204 final dependency list [JobId(58201), JobId(58202), JobId(58203)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.93069783Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"Repair for extent 0 s:2 d:[ClientId(0)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.931386796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"RE:0 Wait for result from repair command 58202:57203","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.931414718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58202 final dependency list [JobId(58201)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.931443946Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"dropping ds1 now!","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:33.931785451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"reconnecting ds1 now!","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:33.931809642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] job id 58202 saw error GenericError(\"bad news, networks are tricky\")","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:33.931995216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] DS Reports error Err(GenericError(\"bad news, networks are tricky\")) on job 58202, DownstairsIO { ds_id: JobId(58202), guest_id: 57203, work: ExtentLiveRepair { dependencies: [JobId(58201)], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:45462, repair_downstairs: [ClientId(0)] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 9 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:33.932039314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] Reports error GenericError(\"bad news, networks are tricky\") on job 58202, DownstairsIO { ds_id: JobId(58202), guest_id: 57203, work: ExtentLiveRepair { dependencies: [JobId(58201)], extent: 0, source_downstairs: ClientId(2), source_repair_address: 127.0.0.1:45462, repair_downstairs: [ClientId(0)] }, state: ClientData([InProgress, InProgress, InProgress]), ack_status: NotAcked, replay: false, data: None, read_response_hashes: [], impacted_blocks: InclusiveRange(ImpactedAddr { extent_id: 0, block: 0 }, ImpactedAddr { extent_id: 0, block: 9 }) }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:33.932077038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] client skip 3 in process jobs because fault","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.932098905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] changed 1 jobs to fault skipped","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.932119204Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) LiveRepair Active Active ds_transition to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.932142041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from LiveRepair to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.932161593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] will exit pm_task, this downstairs Faulted","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:33.932181162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"127.0.0.1:36984: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Faulted)), so we end too","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:33.932202422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa Gone missing, transition from Faulted to Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.93222457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa connection to 127.0.0.1:36984 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.932245075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] 127.0.0.1:36984 task reports connection:false","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.932272215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"28a12560-c07c-41d3-8d35-7e06b0c337aa Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.932292772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 127.0.0.1:36984 task reports offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.932317797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Extent 0 close id:58202 Failed: IO Error: 1 out of 3 downstairs failed to complete this IO","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:33.932339903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:0 Wait for result from NoOp command 58203:57204","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:33.932359782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"spawn_message_receiver saw disconnect, bailing","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:33.932552567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"ds1 negotiate start now!","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:34.933396756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.933450608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"looper":"0","upstairs":1}
{"msg":"[0] Proc runs for 127.0.0.1:36984 in state Faulted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.933474628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet HereIAm { version: 4, upstairs_id: 28a12560-c07c-41d3-8d35-7e06b0c337aa, session_id: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a, gen: 1, read_only: false, encrypted: false, alternate_versions: [] }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.933601179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"[0] upstairs guest_io_ready=TRUE, promote! session 7f1ba66d-4c7a-4444-a654-4b5bfde1244a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.933709722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet PromoteToActive { upstairs_id: 28a12560-c07c-41d3-8d35-7e06b0c337aa, session_id: 7f1ba66d-4c7a-4444-a654-4b5bfde1244a, gen: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.933811683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"negotiate packet RegionInfoPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.933954256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"ds1 negotiate extent versions please now!","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:34.933980977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] downstairs client at 127.0.0.1:36984 has UUID 8dfe8526-b15f-436d-8c37-dad574df0099","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934063724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 10, shift: 9 }, extent_count: 10, uuid: 8dfe8526-b15f-436d-8c37-dad574df0099, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934095111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Returning client:0 UUID:8dfe8526-b15f-436d-8c37-dad574df0099 matches","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934116212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"28a12560-c07c-41d3-8d35-7e06b0c337aa Faulted Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.93413637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"negotiate packet ExtentVersionsPlease","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.93416676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"downstairs":1}
{"msg":"ds1 spawn message receiver now!","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:34.934187677Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) Faulted Active Active ds_transition to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934400535Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from Faulted to LiveRepairReady","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934427145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] new RM replaced this: Some(RegionMetadata { generation: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], flush_numbers: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], dirty: [false, false, false, false, false, false, false, false, false, false] })","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:34.93445218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa Enter Ready for LiveRepair mode","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:34.934472603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934491473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 127.0.0.1:36984 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934629806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"28a12560-c07c-41d3-8d35-7e06b0c337aa LiveRepairReady Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934656774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934676524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:0 Wait for result from reopen command 58204:57205","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934885833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:0 Bailing with error","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:34.934911756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Error After extent 0 repair","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:34.934931402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"Start extent 1 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.93495212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"extent 1 repair has failed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.934972414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"Exit repair at extent 1","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:34.934992377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"LiveRepair final flush submitted","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.935012559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"LiveRepair final flush completed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:34.936845982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Live Repair returns Ok(())","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:34.936871887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.935528527Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 28a12560-c07c-41d3-8d35-7e06b0c337aa (7f1ba66d-4c7a-4444-a654-4b5bfde1244a) LiveRepairReady Active Active ds_transition to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.935590497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] Transition from LiveRepairReady to LiveRepair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.93561181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Live Repair started","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.935631294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"Live Repair main task begins.","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:35.935650477Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"Start Live Repair of extents 0 to 10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.935671879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"Start extent 0 repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.935692017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"task":"repair","upstairs":1}
{"msg":"RE:0 repair extent with ids 58206,58207,58208,58209 deps:[]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.935713032Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:0 close id:58206 queued, notify DS","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.93573239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"RE:0 Wait for result from close command 58206:57207","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.935752278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"upstairs":1}
{"msg":"[0] 58206 final dependency list []","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.935773822Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
{"msg":"[0] 58209 final dependency list [JobId(58206), JobId(58207), JobId(58208)]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:35.935875349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4759,"":"downstairs","upstairs":1}
test dummy_downstairs_tests::protocol_test::test_error_during_live_repair_no_halt ... ok

test result: ok. 351 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 397.39s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_agent-acbf118b39db417b --nocapture`

running 11 tests
{"id":"abc","state":"requested","block_size":4096,"extent_size":4096,"extent_count":100,"encrypted":false,"port_number":1701,"cert_pem":null,"key_pem":null,"root_pem":null}
Sep 22 23:21:35.992 INFO region f7100c32-c286-432a-9d07-f9e1b092608d state: Requested, component: test_harness_datafile
Sep 22 23:21:35.992 INFO region 064d55d6-7fae-45ce-a14e-8a514525e144 state: Requested, component: test_harness_datafile
Sep 22 23:21:35.992 INFO region d7a15ef3-a107-4974-8fb9-03e5d5353ced state: Requested, component: test_harness_datafile
Sep 22 23:21:35.992 INFO region ea20e721-32d0-4fd5-938b-1abf7dd2728d state: Requested, component: test_harness_datafile
Sep 22 23:21:35.992 INFO region fb7a6512-7a38-4a4d-9682-5d11d0d740e8 state: Requested, component: test_harness_datafile
Sep 22 23:21:35.995 INFO region f7100c32-c286-432a-9d07-f9e1b092608d state: Requested -> Created, component: test_harness_datafile
Sep 22 23:21:35.995 INFO region ea20e721-32d0-4fd5-938b-1abf7dd2728d state: Requested -> Created, component: test_harness_datafile
Sep 22 23:21:35.995 INFO region fb7a6512-7a38-4a4d-9682-5d11d0d740e8 state: Requested -> Tombstoned, component: test_harness_datafile
test model::test::basic ... ok
test test::test_collect_behaviour ... ok
Sep 22 23:21:35.995 INFO region 064d55d6-7fae-45ce-a14e-8a514525e144 state: Requested -> Tombstoned, component: test_harness_datafile
Sep 22 23:21:35.995 INFO region d7a15ef3-a107-4974-8fb9-03e5d5353ced state: Requested -> Created, component: test_harness_datafile
Sep 22 23:21:35.996 INFO region dcfce67e-c4ff-48cd-9963-6b8373ecd4b6 state: Requested, component: test_harness_datafile
Sep 22 23:21:35.996 INFO creating missing downstairs instance downstairs-d7a15ef3-a107-4974-8fb9-03e5d5353ced
Sep 22 23:21:35.996 INFO creating missing downstairs instance downstairs-f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.996 INFO creating missing downstairs instance downstairs-ea20e721-32d0-4fd5-938b-1abf7dd2728d
Sep 22 23:21:35.996 INFO region dcfce67e-c4ff-48cd-9963-6b8373ecd4b6 state: Requested -> Failed, component: test_harness_datafile
Sep 22 23:21:35.996 INFO region 5a285992-c4f9-4c6c-8322-87d73bb5c18f state: Requested, component: test_harness_datafile
Sep 22 23:21:35.996 INFO ok, have oxide/crucible/downstairs:downstairs-ea20e721-32d0-4fd5-938b-1abf7dd2728d
Sep 22 23:21:35.996 INFO region 5a285992-c4f9-4c6c-8322-87d73bb5c18f state: Requested -> Created, component: test_harness_datafile
test datafile::test::test_stat_parsing ... ok
Sep 22 23:21:35.996 INFO ok, have oxide/crucible/downstairs:downstairs-f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.996 INFO ok, have oxide/crucible/downstairs:downstairs-d7a15ef3-a107-4974-8fb9-03e5d5353ced
Sep 22 23:21:35.996 INFO reconfig required, no property group
Sep 22 23:21:35.996 INFO reconfig required, no property group
Sep 22 23:21:35.996 INFO reconfig required, no property group
test test::test_smf_datafile_race_region ... ok
test test::test_smf_region_failed ... ok
Sep 22 23:21:35.996 INFO creating config property group
Sep 22 23:21:35.996 INFO creating config property group
Sep 22 23:21:35.996 INFO creating config property group
test test::test_smf_region_create_then_destroy ... ok
Sep 22 23:21:35.996 INFO reconfiguring oxide/crucible/downstairs:downstairs-d7a15ef3-a107-4974-8fb9-03e5d5353ced
Sep 22 23:21:35.996 INFO reconfiguring oxide/crucible/downstairs:downstairs-f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.996 INFO reconfiguring oxide/crucible/downstairs:downstairs-ea20e721-32d0-4fd5-938b-1abf7dd2728d
Sep 22 23:21:35.996 INFO creating missing downstairs instance downstairs-5a285992-c4f9-4c6c-8322-87d73bb5c18f
Sep 22 23:21:35.996 INFO ok, have oxide/crucible/downstairs:downstairs-5a285992-c4f9-4c6c-8322-87d73bb5c18f
Sep 22 23:21:35.996 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpykMspN/regions/f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.996 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpk5bUdb/regions/d7a15ef3-a107-4974-8fb9-03e5d5353ced
Sep 22 23:21:35.996 INFO reconfig required, no property group
Sep 22 23:21:35.996 INFO creating config property group
Sep 22 23:21:35.996 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:21:35.996 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:21:35.996 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmp5GQNwb/regions/ea20e721-32d0-4fd5-938b-1abf7dd2728d
Sep 22 23:21:35.996 INFO reconfiguring oxide/crucible/downstairs:downstairs-5a285992-c4f9-4c6c-8322-87d73bb5c18f
Sep 22 23:21:35.996 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:21:35.996 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:21:35.996 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmp1EK8D2/regions/5a285992-c4f9-4c6c-8322-87d73bb5c18f
Sep 22 23:21:35.996 INFO commit
Sep 22 23:21:35.996 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:21:35.996 INFO commit
Sep 22 23:21:35.996 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:21:35.996 INFO ok!
Sep 22 23:21:35.996 INFO commit
Sep 22 23:21:35.996 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:21:35.996 INFO ok!
Sep 22 23:21:35.996 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:21:35.996 INFO ok!
Sep 22 23:21:35.996 INFO commit
Sep 22 23:21:35.996 INFO region d7a15ef3-a107-4974-8fb9-03e5d5353ced state: Created -> Tombstoned, component: test_harness_datafile
Sep 22 23:21:35.996 INFO ok!
Sep 22 23:21:35.997 INFO creating missing downstairs instance downstairs-ea20e721-32d0-4fd5-938b-1abf7dd2728d
Sep 22 23:21:35.997 INFO ok, have oxide/crucible/downstairs:downstairs-ea20e721-32d0-4fd5-938b-1abf7dd2728d
Sep 22 23:21:35.997 INFO creating region 5a285992-c4f9-4c6c-8322-87d73bb5c18f snapshot 789d4889-da87-4e0a-b406-52cd4a361df2, component: test_snapshot_interface
Sep 22 23:21:35.997 INFO creating region f7100c32-c286-432a-9d07-f9e1b092608d snapshot a285abac-db97-457e-987b-06a4be752bd5, component: test_snapshot_interface
Sep 22 23:21:35.997 INFO reconfig required, no property group
Sep 22 23:21:35.997 INFO creating config property group
Sep 22 23:21:35.997 INFO reconfiguring oxide/crucible/downstairs:downstairs-ea20e721-32d0-4fd5-938b-1abf7dd2728d
Sep 22 23:21:35.997 INFO creating region 5a285992-c4f9-4c6c-8322-87d73bb5c18f snapshot 789d4889-da87-4e0a-b406-52cd4a361df2 dir "/tmp/.tmp1EK8D2/regions/5a285992-c4f9-4c6c-8322-87d73bb5c18f/.zfs/snapshot/789d4889-da87-4e0a-b406-52cd4a361df2", component: test_snapshot_interface
Sep 22 23:21:35.997 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmp5GQNwb/regions/ea20e721-32d0-4fd5-938b-1abf7dd2728d
Sep 22 23:21:35.997 INFO creating region f7100c32-c286-432a-9d07-f9e1b092608d snapshot a285abac-db97-457e-987b-06a4be752bd5 dir "/tmp/.tmpykMspN/regions/f7100c32-c286-432a-9d07-f9e1b092608d/.zfs/snapshot/a285abac-db97-457e-987b-06a4be752bd5", component: test_snapshot_interface
Sep 22 23:21:35.997 INFO disabling downstairs instance: downstairs-d7a15ef3-a107-4974-8fb9-03e5d5353ced (instance states: (Some(Online), None))
Sep 22 23:21:35.997 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:21:35.997 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:21:35.997 INFO commit
Sep 22 23:21:35.997 INFO ok!
Sep 22 23:21:35.997 INFO requesting running snapshot 5a285992-c4f9-4c6c-8322-87d73bb5c18f-789d4889-da87-4e0a-b406-52cd4a361df2 state: Requested, component: test_harness_datafile
Sep 22 23:21:35.997 INFO requesting running snapshot f7100c32-c286-432a-9d07-f9e1b092608d-a285abac-db97-457e-987b-06a4be752bd5 state: Requested, component: test_harness_datafile
test test::test_smf_region ... ok
Sep 22 23:21:35.997 DEBG found expected downstairs instance: downstairs-5a285992-c4f9-4c6c-8322-87d73bb5c18f
Sep 22 23:21:35.997 DEBG found expected downstairs instance: downstairs-f7100c32-c286-432a-9d07-f9e1b092608d
test test::test_smf_region_bounce_idempotent ... ok
Sep 22 23:21:35.997 DEBG do not need to reconfigure oxide/crucible/downstairs:downstairs-5a285992-c4f9-4c6c-8322-87d73bb5c18f
Sep 22 23:21:35.997 DEBG do not need to reconfigure oxide/crucible/downstairs:downstairs-f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.997 INFO creating missing snapshot instance snapshot-5a285992-c4f9-4c6c-8322-87d73bb5c18f-789d4889-da87-4e0a-b406-52cd4a361df2
Sep 22 23:21:35.997 INFO creating missing snapshot instance snapshot-f7100c32-c286-432a-9d07-f9e1b092608d-a285abac-db97-457e-987b-06a4be752bd5
Sep 22 23:21:35.997 INFO ok, have oxide/crucible/downstairs:snapshot-5a285992-c4f9-4c6c-8322-87d73bb5c18f-789d4889-da87-4e0a-b406-52cd4a361df2
Sep 22 23:21:35.997 INFO ok, have oxide/crucible/downstairs:snapshot-f7100c32-c286-432a-9d07-f9e1b092608d-a285abac-db97-457e-987b-06a4be752bd5
Sep 22 23:21:35.997 INFO reconfig required, no property group
Sep 22 23:21:35.997 INFO reconfig required, no property group
Sep 22 23:21:35.997 INFO reconfiguring oxide/crucible/downstairs:snapshot-5a285992-c4f9-4c6c-8322-87d73bb5c18f-789d4889-da87-4e0a-b406-52cd4a361df2
Sep 22 23:21:35.997 INFO reconfiguring oxide/crucible/downstairs:snapshot-f7100c32-c286-432a-9d07-f9e1b092608d-a285abac-db97-457e-987b-06a4be752bd5
Sep 22 23:21:35.997 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmp1EK8D2/regions/5a285992-c4f9-4c6c-8322-87d73bb5c18f/.zfs/snapshot/789d4889-da87-4e0a-b406-52cd4a361df2
Sep 22 23:21:35.997 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpykMspN/regions/f7100c32-c286-432a-9d07-f9e1b092608d/.zfs/snapshot/a285abac-db97-457e-987b-06a4be752bd5
Sep 22 23:21:35.997 INFO ensure port SCF_TYPE_COUNT 1001
Sep 22 23:21:35.997 INFO ensure port SCF_TYPE_COUNT 1001
Sep 22 23:21:35.997 INFO ensure mode SCF_TYPE_ASTRING ro
Sep 22 23:21:35.997 INFO ensure mode SCF_TYPE_ASTRING ro
Sep 22 23:21:35.997 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:21:35.997 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:21:35.997 INFO commit
Sep 22 23:21:35.997 INFO commit
Sep 22 23:21:35.997 INFO ok!
Sep 22 23:21:35.997 INFO ok!
Sep 22 23:21:35.997 INFO region 5a285992-c4f9-4c6c-8322-87d73bb5c18f running snapshot 789d4889-da87-4e0a-b406-52cd4a361df2 state: Requested -> Created, component: test_harness_datafile
Sep 22 23:21:35.997 INFO removing running snapshot f7100c32-c286-432a-9d07-f9e1b092608d-a285abac-db97-457e-987b-06a4be752bd5, component: test_harness_datafile
Sep 22 23:21:35.998 INFO creating missing downstairs instance downstairs-5a285992-c4f9-4c6c-8322-87d73bb5c18f
Sep 22 23:21:35.998 INFO ok, have oxide/crucible/downstairs:downstairs-5a285992-c4f9-4c6c-8322-87d73bb5c18f
Sep 22 23:21:35.998 DEBG found expected downstairs instance: downstairs-f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.998 INFO disabling snapshot instance: snapshot-f7100c32-c286-432a-9d07-f9e1b092608d-a285abac-db97-457e-987b-06a4be752bd5 (instance states: (Some(Online), None))
Sep 22 23:21:35.998 INFO reconfig required, no property group
Sep 22 23:21:35.998 INFO creating config property group
Sep 22 23:21:35.998 INFO reconfiguring oxide/crucible/downstairs:downstairs-5a285992-c4f9-4c6c-8322-87d73bb5c18f
Sep 22 23:21:35.998 DEBG do not need to reconfigure oxide/crucible/downstairs:downstairs-f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.998 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmp1EK8D2/regions/5a285992-c4f9-4c6c-8322-87d73bb5c18f
Sep 22 23:21:35.998 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:21:35.998 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:21:35.998 INFO commit
Sep 22 23:21:35.998 INFO creating missing downstairs instance downstairs-f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.998 INFO ok, have oxide/crucible/downstairs:downstairs-f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.998 INFO ok!
Sep 22 23:21:35.998 INFO reconfig required, no property group
Sep 22 23:21:35.998 INFO creating config property group
Sep 22 23:21:35.998 INFO reconfiguring oxide/crucible/downstairs:downstairs-f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.998 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmpykMspN/regions/f7100c32-c286-432a-9d07-f9e1b092608d
Sep 22 23:21:35.998 INFO creating missing snapshot instance snapshot-5a285992-c4f9-4c6c-8322-87d73bb5c18f-789d4889-da87-4e0a-b406-52cd4a361df2
Sep 22 23:21:35.998 INFO ensure port SCF_TYPE_COUNT 1000
Sep 22 23:21:35.998 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:21:35.998 INFO ok, have oxide/crucible/downstairs:snapshot-5a285992-c4f9-4c6c-8322-87d73bb5c18f-789d4889-da87-4e0a-b406-52cd4a361df2
Sep 22 23:21:35.998 INFO commit
Sep 22 23:21:35.998 INFO reconfig required, no property group
Sep 22 23:21:35.998 INFO reconfiguring oxide/crucible/downstairs:snapshot-5a285992-c4f9-4c6c-8322-87d73bb5c18f-789d4889-da87-4e0a-b406-52cd4a361df2
Sep 22 23:21:35.998 INFO ok!
Sep 22 23:21:35.998 INFO ensure directory SCF_TYPE_ASTRING /tmp/.tmp1EK8D2/regions/5a285992-c4f9-4c6c-8322-87d73bb5c18f/.zfs/snapshot/789d4889-da87-4e0a-b406-52cd4a361df2
Sep 22 23:21:35.998 INFO ensure port SCF_TYPE_COUNT 1001
Sep 22 23:21:35.998 INFO ensure mode SCF_TYPE_ASTRING ro
Sep 22 23:21:35.998 INFO ensure address SCF_TYPE_ASTRING 127.0.0.1
Sep 22 23:21:35.998 INFO commit
Sep 22 23:21:35.998 INFO ok!
test test::test_smf_datafile_race_running_snapshots ... ok
test test::test_smf_running_snapshot ... ok
test tests::test_crucible_agent_openapi ... ok

test result: ok. 11 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_agent_client-6f3900e8033b57ec --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_client_types-5ba9f9d411803900 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_common-9a32809bdbdf85c4 --nocapture`

running 2 tests
test region::test::test_basic_region ... ok
test region::test::test_region_validate_io ... ok

test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_control_client-d0a58354872d46d9 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_downstairs-dce67baac661a5f4 --nocapture`

running 156 tests
test dump::test::color_compare ... ok
test dump::test::color_compare_one ... ok
test dump::test::color_compare_red0 ... ok
test dump::test::color_compare_red01 ... ok
test dump::test::color_compare_red01_2 ... ok
test dump::test::color_compare_red02_2 ... ok
test dump::test::color_compare_red02 ... ok
test dump::test::color_compare_red1 ... ok
test dump::test::color_compare_red12 ... ok
test dump::test::color_compare_red12_2 ... ok
test dump::test::color_compare_red2 ... ok
test dump::test::color_compare_two ... ok
test dump::test::color_compare_two_red0 ... ok
test dump::test::color_compare_two_red1 ... ok
test region::test::copy_path_basic ... ok
thread 'region::test::bad_import_region' panicked at 'called `Result::unwrap()` on an `Err` value: Error open "/tmp/12345678-1111-2222-3333-123456789999/notadir/region.json": file not found opening region config "/tmp/12345678-1111-2222-3333-123456789999/notadir/region.json"', downstairs/src/region.rs:3466:10
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
{{""msg"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible":"current number of open files limit 65536 is already the maximum","level":,"v":0,"name":"crucible","level":3030{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.073836316Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time":"2023-09-22T23:21:36.074315297Z","hostname":","pid":4766ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
}
{{"msg":""msg":"Created new region file \"Created new region file /tmp/.tmpHePGDE/region.json\"\""/tmp/.tmpJVZno2/region.json,"\"v"":0,","v"name"::"0crucible",","name"level"::"30crucible","level":30,"time":"2023-09-22T23:21:36.074644584Z",,""time":hostname":""2023-09-22T23:21:36.074647494Z"ip-10-150-1-55.us-west-2.compute.internal",,""hostname"pid"::"4766}
ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"2023-09-22T23:21:36.074216787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpzvev7V/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.075337309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"2023-09-22T23:21:36.073883476Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpX75Ahj/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.075520272Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.078641772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpoRNhEc/region.json\"","v":0,"name"test region::test::bad_import_region - should panic ... :"ok
crucible","level":30,"time":"2023-09-22T23:21:36.078740094Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"2023-09-22T23:21:36.073991861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpFHgHPW/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.078852578Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"2023-09-22T23:21:36.074048022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpho5Xja/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.079038887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"2023-09-22T23:21:36.074033219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpJGYNMa/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.080358998Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.081122797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp05J1Ma/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.081173367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.091192214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.091253202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.091292163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::duplicate_context_insert ... ok
test region::test::encryption_context ... ok
test region::test::extent_dir_max ... ok
test region::test::extent_dir_basic ... ok
test region::test::extent_dir_min ... ok
thread 'region::test::extent_io_bad_block' panicked at 'called `Result::unwrap()` on an `Err` value: OffsetInvalid', downstairs/src/region.rs:3506:53
thread 'region::test::extent_io_invalid_block_buf' panicked at 'called `Result::unwrap()` on an `Err` value: DataLenUnaligned', downstairs/src/region.rs:3516:52
test region::test::extent_io_bad_block - should panic ... ok
test region::test::extent_io_invalid_block_buf - should panic ... ok
thread 'region::test::extent_io_invalid_large' panicked at 'called `Result::unwrap()` on an `Err` value: DataLenUnaligned', downstairs/src/region.rs:3526:51
thread 'region::test::extent_io_non_aligned_large' panicked at 'called `Result::unwrap()` on an `Err` value: DataLenUnaligned', downstairs/src/region.rs:3486:51
test region::test::extent_io_invalid_large - should panic ... ok
test region::test::extent_io_non_aligned_large - should panic ... ok
thread 'region::test::extent_io_non_aligned_small' panicked at 'called `Result::unwrap()` on an `Err` value: DataLenUnaligned', downstairs/src/region.rs:3496:51
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","test region::test::copy_extent_dir ... level"ok
:30,"time":"2023-09-22T23:21:36.099286301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpiuTKWT/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.099340117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::extent_name_basic ... ok
test region::test::extent_name_basic_ext ... ok
test region::test::extent_io_non_aligned_small - should panic ... ok
test region::test::extent_name_basic_ext_shm ... ok
test region::test::extent_name_basic_ext_wal ... ok
test region::test::extent_name_basic_three ... ok
test region::test::extent_name_basic_two ... ok
test region::test::extent_name_max ... ok
test region::test::extent_io_valid ... ok
test region::test::extent_name_min ... ok
test region::test::extent_path_max ... ok
test region::test::close_extent ... ok
test region::test::extent_path_mid_lo ... ok
EXT  BLOCKS GEN0  FL0  D0
  0 000-009test region::test::extent_path_mid_hi ... ok
    0    0 test region::test::extent_path_min ... ok
  F
  1 010-019    0    0   F
Max gen: 0,  Max flush: 0
test region::test::extent_path_three ... ok
test region::test::copy_extent_dir_twice ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.103120305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpi6lJFh/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.10317306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.10323207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp77DdeC/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.103285698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.103602645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpN2tu9r/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.10365516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.103763278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{{"msg":"Opened existing region file "\"msg":"/tmp/.tmpr8RcYx/region.json\"","v":0current number of open files limit 65536 is already the maximum,""name":","crucible"v",:"level0":,"30name":"crucible","level":30,"time":"2023-09-22T23:21:36.103819715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"2023-09-22T23:21:36.103824073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.103878073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.103920114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::new_region ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30test region::test::dump_a_region ... ,ok
"time":"2023-09-22T23:21:36.104357351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmp6hm9W2/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.104419169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::region_bad_database_read_version_high ... ok
test region::test::new_existing_region ... ok
test region::test::region_bad_database_read_version_low ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.10510606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpvxVNj7/region.json\"","v":0,"name":"crucible","level":30{"msg":","time"current number of open files limit 65536 is already the maximum":","2023-09-22T23:21:36.105162454Z"v":,"0hostname":","name":"ip-10-150-1-55.us-west-2.compute.internal"crucible",",pid"":level4766":}30
{{"msg":""Database read version 1"msg",:""v":0,"name":current number of open files limit 65536 is already the maximum""crucible",","v"level":,"0:,time""name:"":30"2023-09-22T23:21:36.105192994Z"crucible",,""level"hostname:":"30ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"2023-09-22T23:21:36.105218709Z{","hostname"":,msg"""time:"":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:21:36.105222246ZCreated new region file ""\",,"/tmp/.tmp74dTT1/region.json"\"hostname""pid,:""v""::ip-10-150-1-55.us-west-2.compute.internal0"4766,,""}pidname""::4766"
crucible}"
,"level":30{"msg":"Opened existing region file \"/tmp/.tmpppQckI/region.json\"","v":0,"name":"crucible","level",:"30time":"2023-09-22T23:21:36.105265968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"2023-09-22T23:21:36.105278105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.105309895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.105450646Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4766}"
msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.105501117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmptGTFWj/region.json\"","v":0,"name":"crucible","level":30,"time":"test region::test::region_bad_database_write_version_high ... 2023-09-22T23:21:36.105551894Zok
","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpi2WW6m/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.105606961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::region_bad_database_write_version_low ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.105969565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.106016608Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.106051318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.106341193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpNnqPXL/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.106437533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.106516805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpfQyUMp/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.106564831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.108353435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.108725575Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.108776756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::multiple_context ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.113666691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpF6IEvH/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.113768458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.11401903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.114051155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.114073953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.115188801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{{""msg":"msg":"Opened existing region file \"/tmp/.tmp74dTT1/region.json\""current number of open files limit 65536 is already the maximum,""v":0,","name"v"::"0crucible",,""name"level"::"30crucible","level":30,"time":"2023-09-22T23:21:36.115244506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{,""msgtime"":":"Database read version 1","v":02023-09-22T23:21:36.115248095Z","name":","crucible","hostname"level"::30"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"2023-09-22T23:21:36.115278141Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
"msg":{""Database read version 1"msg":,"v":"0,"Database write version 1"name",:""v":crucible"0,,""level"name"::"30crucible","level":30,,""time"time":":"2023-09-22T23:21:36.115312948Z"2023-09-22T23:21:36.115315234Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"",pid"":pid"4766:4766}
}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.115359715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
cp "/tmp/.tmpfQyUMp/00/000/000" to "/tmp/.tmpfQyUMp/00/000/001.copy/001"
Recreate "/tmp/.tmpfQyUMp/00/000/001.db-shm"
Recreate "/tmp/.tmpfQyUMp/00/000/001.db-wal"
{"msg":"Extent 1 found replacement dir, finishing replacement","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.116588583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Copy files from \"/tmp/.tmpfQyUMp/00/000/001.replace\" in \"/tmp/.tmpfQyUMp/00/000\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.116642263Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Remove old file \"/tmp/.tmpfQyUMp/00/000/001.db-shm\" as there is no replacement","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.116811892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Remove old file \"/tmp/.tmpfQyUMp/00/000/001.db-wal\" as there is no replacement","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.116864738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Move directory  \"/tmp/.tmpfQyUMp/00/000/001.replace\" to \"/tmp/.tmpfQyUMp/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.116906856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Extent 1 found replacement dir, finishing replacement","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.118701224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Copy files from \"/tmp/.tmpNnqPXL/00/000/001.replace\" in \"/tmp/.tmpNnqPXL/00/000\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.118762146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Move directory  \"/tmp/.tmpNnqPXL/00/000/001.replace\" to \"/tmp/.tmpNnqPXL/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.119009645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::reopen_extent_cleanup_replay_short ... ok
           Extent 2
GEN             0        0 
FLUSH_ID        0        0 
DIRTY                      

BLOCK  D0 D1  C0 C1 DIFF 
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.120937102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.120987583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.121023383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.121246472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpntsZot/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.121317377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::reopen_extent_cleanup_one ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.12245947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmprdPlCZ/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.122518132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
EXT  BLOCKS GEN0 GEN1  FL0 FL1  D0 D1
  0 000-009    0    0    0   0   F  F
  1 010-019    0    0    0   0   F  F
Max gen: 0,  Max flush: 0
test region::test::reopen_extent_cleanup_replay ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.124261041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpNuvjXd/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.124395154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::region_create_drop_open ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.125684629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp8sn7MW/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.125809877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::dump_two_region ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.127141174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpomLMVu/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.127193498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::reopen_extent_cleanup_two ... ok
{"msg":"Failed write hash validation","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:36.128794397Z"{,"hostname":"ip-10-150-1-55.us-west-2.compute.internal""msg",:""pid":4766current number of open files limit 65536 is already the maximum"},"
v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.128853368Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpgvVgcF/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.128915047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.129166662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.129212218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.129248782Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_bad_hash_bad ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.130934598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpQNdK4B/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.130989566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.131084482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.131134672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.131168085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_blank_block_read_ok ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.133609299Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpy4Quhy/region.json\"","v":0,"name":"crucible","level":30,020 "time":  A"  A   A  A2023-09-22T23:21:36.133741631Z"     ,
test region::test::reopen_all_extents ... ok
"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.133971815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.13402186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.134062006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.13434635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpXjT7Kw/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.134424307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::reopen_extent_no_replay_readonly ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.137675961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpxFRMrp/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.137732542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_big_write ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.139374426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.139422874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.139456016Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_extent_write_flush_close ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.139754821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpMj5GG3/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.139888885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.140270085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpuWtzyh/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.14033568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_extent_close_reopen_flush_close ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.142139758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpH48PpE/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.142275644Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_flush_extent_limit_base ... ok
test region::test::test_flush_extent_limit_too_large ... ok
021   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.143896924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpqrtt1Y/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.143960308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.144221745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.144271774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"{crucible","level":30"msg":"current number of open files limit 65536 is already the maximum","v,"":time":"0,"2023-09-22T23:21:36.144305536Z"name":,""crucible"hostname":,""level":ip-10-150-1-55.us-west-2.compute.internal30","pid":4766}
,"time":"2023-09-22T23:21:36.144328701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpSuumQF/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.144428944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_flush_extent_limit_end ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.14722755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp6x9BQd/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.147282279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.148717548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_fully_rehash_and_clean_does_not_mark_blocks_as_written ... ok
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.148961659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.149008378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.14941733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp3ADSVm/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.149542943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_ok_hash_ok ... ok
test region::test::test_fully_rehash_marks_blocks_unwritten_if_data_never_hit_disk ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.150272243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp3gtgmK/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.15043902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.150608764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpyPH46t/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.150749246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
022   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.153789989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.153881182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.15392743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.157467261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.157517978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.157558747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Send flush to extent limit 0
test region::test::test_read_multiple_disjoint_large_contiguous ... ok
extent 0 should not be dirty now
verify 1 still dirty
verify 2 still dirty
verify 3 still dirty
verify 4 still dirty
verify 5 still dirty
verify 6 still dirty
verify 7 still dirty
{"msg":"current number of open files limit 65536 is already the maximum",verify 8 still dirty
"v":0,"name":"crucible","level":30verify 9 still dirty
,"time":"2023-09-22T23:21:36.160191505Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Send flush to extent limit 1
023   A  A   A  A     
{"msg":"Created new region file \"/tmp/.tmpfOWcq5/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.160273592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_read_multiple_disjoint_none_contiguous ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.160451413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.160526167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.160565679Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
extent 1 should not be dirty now
verify 2 still dirty
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.161294124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpnAVzN7/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.161363205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
verify 3 still dirty
verify 4 still dirty
verify 5 still dirty
verify 6 still dirty
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 2
extent 2 should not be dirty now
test region::test::test_read_single_large_contiguous ... ok
verify 3 still dirty
verify 4 still dirty
verify 5 still dirty
verify 6 still dirty
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 3
extent 3 should not be dirty now
verify 4 still dirty
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.163612267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpi1O2O9/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.163790325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
verify 5 still dirty
verify 6 still dirty
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 4
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.164788192Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"extent 4 should not be dirty now
,"pid":4766}
verify 5 still dirty
verify 6 still dirty
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 5
test region::test::test_read_single_large_contiguous_span_extents ... ok
{extent 5 should not be dirty now
"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.165287477Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.165349072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.165522537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpPcSvx5/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.165599528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
verify 6 still dirty
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 6
extent 6 should not be dirty now
verify 7 still dirty
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 7
test region::test::test_region_open_removes_partial_writes ... ok
extent 7 should not be dirty now
verify 8 still dirty
verify 9 still dirty
Send flush to extent limit 8
extent 8 should not be dirty now
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.167357611Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpvA8mjM/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.167497669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
verify 9 still dirty
Send flush to extent limit 9
extent 9 should not be dirty now
024   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.170073987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.170123182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.170158912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_flush_extent_limit_walk_it_off ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.172985004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.173034143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.173071915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.17336581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmperRgma/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.173442203Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_write_multiple_disjoint_large_contiguous ... ok
025   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.175962352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.17600405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30test region::test::test_write_multiple_disjoint_none_contiguous ... ok
,"time":"2023-09-22T23:21:36.176034295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.176146928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpzWh5Jq/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.176240964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.176709025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp8g1tlB/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.176787181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_write_single_large_contiguous ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.178491546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpiH1UWN/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.178544148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.179705733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.17982018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.179884208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Total size: 15360
test region::test::test_write_single_large_contiguous_span_extents ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.182035788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmplN2WTP/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.182175463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
026   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.182930097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.182983574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.183022031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_write_unwritten_big_write ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.184787953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpqIRCQc/region.json\"","v":0,"name"buffer size:2048
:"crucible","level":30,"time":"2023-09-22T23:21:36.184846764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Read eid: 0, 0 offset: Block { value: 0, shift: 9 }
Read eid: 0, 1 offset: Block { value: 1, shift: 9 }
Read eid: 0, 2 offset: Block { value: 2, shift: 9 }
Read eid: 0, 3 offset: Block { value: 3, shift: 9 }
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.185621531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.185673529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.18571042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Read a region, append
Read a region, append
Read a region, append
Read a region, append
test region::test::test_write_unwritten_big_write_partial_0 ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.187134414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpAqU3tu/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.187189117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Total size: 20480
test region::test::test_write_unwritten_big_write_partial_1 ... ok
test region::test::test_write_unwritten_big_write_partial_final ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.189419629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp5xP0uT/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.189633265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.189877853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpPV6kWM/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.18994392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
027   A  A   A  A     
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.190975173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.191295805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.191336435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.193709309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.1940038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.194050252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_write_unwritten_big_write_partial_sparse ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.1973654Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpauX5Ae/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.197517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_write_unwritten_multiple_disjoint_large_contiguous ... ok
test region::test::test_write_unwritten_multiple_disjoint_none_contiguous ... ok
test region::test::test_write_unwritten_single_large_contiguous ... {ok
"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.198391884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
028   A  A   A  A     
{"msg":"Created new region file \"/tmp/.tmpuaYM4G/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.198532018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible",","level"time"::"302023-09-22T23:21:36.198582864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::validate_repair_files_also_good ... ok,
"time":"2023-09-22T23:21:36.198608693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmppqa7nG/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.19866426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.19872916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.198771561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::validate_repair_files_duplicate ... ok
test region::test::validate_repair_files_duplicate_pair ... ok
test region::test::validate_repair_files_empty ... ok
test region::test::validate_repair_files_good ... ok
test region::test::validate_repair_files_not_good_enough ... ok
test region::test::validate_repair_files_offbyon ... ok
test region::test::validate_repair_files_quad_duplicate ... ok
test region::test::test_write_unwritten_single_large_contiguous_span_extents ... ok
test region::test::validate_repair_files_too_good ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.201850869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpSZM0Ba/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.20197961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.20211789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpgmUT7T/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.202165084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_write_unwritten_when_empty ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.202707857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.20279605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.2028348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.202948335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpMUpUBZ/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.203094809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_write_unwritten_when_written ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.204023854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpjvZIIZ/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.204073114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test region::test::test_write_unwritten_when_written_flush ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.20539895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpx0FzWJ/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.205561943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
029   A  A   A  A     
files: ["001", "001.db", "001.db-shm", "001.db-wal"]
test repair::test::extent_expected_files_fail ... ok
test repair::test::extent_expected_files_fail_two ... ok
test repair::test::extent_expected_files ... ok
files: ["001", "001.db"]
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.211369406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpgbUOaV/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.211422825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.211769379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpGKg4P3/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.211825687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test repair::test::extent_expected_files_short ... ok
test region::test::dump_extent ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.21344266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpixzTQ1/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.213620033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.214400245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
files: ["001", "001.db"]
{"msg":"Created new region file \"/tmp/.tmpBn4xhF/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.21454597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test repair::test::extent_expected_files_short_with_close ... ok
{"msg":"1002 job Flush for connection UpstairsConnection { upstairs_id: 46c133d9-b38b-4055-adce-337c31923de1, session_id: 414a6c2c-a324-4ae8-9ffd-3c12d2b3e1c0, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.218503863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::job_dep_not_satisfied ... ok
test test::jobs_extent_close ... ok
test test::jobs_extent_flush_close ... ok
test test::jobs_extent_live_noop ... ok
test test::jobs_extent_live_reopen ... ok
test test::jobs_extent_live_repair ... ok
test repair::test::test_crucible_repair_openapi ... ok
test test::jobs_independent ... ok
test test::jobs_write_unwritten ... ok
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: c02a08ce-458e-4f61-907f-97a9103eb0dd, session_id: d3f895e5-eda6-4e78-9662-969601129dc3, gen: 0 } waiting on 2 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.227825294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Import file_size: 51200  Extent size: 5120  Needed extents: 10
Region already large enough for image
Importing "/tmp/.tmpSz1cfO/random_data" to region
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: c02a08ce-458e-4f61-907f-97a9103eb0dd, session_id: d3f895e5-eda6-4e78-9662-969601129dc3, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.228411028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: 9cb730fb-14c4-4470-a37f-0a6ddcb7f6c3, session_id: ded622ab-8422-4ba2-9fb6-09cfcaa27eb4, gen: 0 } waiting on 2 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.228579437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Import file_size: 51200  Extent size: 5120  Needed extents: 10
Region already large enough for image
Importing "/tmp/.tmpY1QcCg/random_data" to region
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: 9cb730fb-14c4-4470-a37f-0a6ddcb7f6c3, session_id: ded622ab-8422-4ba2-9fb6-09cfcaa27eb4, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.229630439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::out_of_order_arrives_after_1001_completes ... ok
test test::out_of_order_arrives_after_first_do_work ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.231210102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpey9JDt/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.231370915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Import file_size: 51300  Extent size: 5120  Needed extents: 11
Extending region to fit image
Import file_size: 51100  Extent size: 5120  Needed extents: 10
Region already large enough for image
Importing "/tmp/.tmpBIrDZy/random_data" to region
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: aef73505-1958-4799-9918-432738d9b385, session_id: 7f0ac549-00a1-42ce-8018-190ab0852381, gen: 0 } waiting on 2 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.233062231Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Importing "/tmp/.tmpa7KOZ6/random_data" to region
{"msg":"1003 job Read for connection UpstairsConnection { upstairs_id: aef73505-1958-4799-9918-432738d9b385, session_id: 7f0ac549-00a1-42ce-8018-190ab0852381, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.234644288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::out_of_order_arrives_after_first_push_next_jobs ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.236735647Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpO3n5HR/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.23690483Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.237181952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpey9JDt/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.23723138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.23726674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.237298733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: ec4db1ef-26ef-4452-8c2f-4e84fe9417d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.240324288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.240373443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 6425146f-9763-4b8e-9214-3a0aa05ff047, session_id: 91dc2a17-915b-4d94-a7a6-c9c413ea5a19, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.240407488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 6425146f-9763-4b8e-9214-3a0aa05ff047, session_id: 91dc2a17-915b-4d94-a7a6-c9c413ea5a19, gen: 10 } to UpstairsConnection { upstairs_id: 6425146f-9763-4b8e-9214-3a0aa05ff047, session_id: 91dc2a17-915b-4d94-a7a6-c9c413ea5a19, gen: 10 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.240681256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Signaling to UpstairsConnection { upstairs_id: 6425146f-9763-4b8e-9214-3a0aa05ff047, session_id: 91dc2a17-915b-4d94-a7a6-c9c413ea5a19, gen: 10 } thread that UpstairsConnection { upstairs_id: 6425146f-9763-4b8e-9214-3a0aa05ff047, session_id: 91dc2a17-915b-4d94-a7a6-c9c413ea5a19, gen: 10 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.240734531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Crucible Downstairs promoting UpstairsConnection { upstairs_id: 6425146f-9763-4b8e-9214-3a0aa05ff047, session_id: 91dc2a17-915b-4d94-a7a6-c9c413ea5a19, gen: 10 } to active, discarding 1 jobs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.240776717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 6425146f-9763-4b8e-9214-3a0aa05ff047, session_id: 91dc2a17-915b-4d94-a7a6-c9c413ea5a19, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.240815056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.242430593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpO3n5HR/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.242481567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30test test::test_complete_work_can_see_none ... ,ok
"time":"2023-09-22T23:21:36.2425166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.242556646Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.243232582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: ca00fd14-756b-4b6c-995a-1f0dc95ba49d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.243340485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30{"msg":"Created new region file ,\""time":"/tmp/.tmpICuwkB/region.json\"",2023-09-22T23:21:36.243381296Z""v",:"0hostname,"":"name":"crucible","ip-10-150-1-55.us-west-2.compute.internal"level",:"30pid":4766}
{"msg":","time":"2023-09-22T23:21:36.243400952Z"UpstairsConnection { upstairs_id: 64733bc8-0e54-426a-b284-8ee7a1092e12, session_id: c328a90a-8f5c-4369-a069-ea688b82e263, gen: 10 } is now active (read-write),"","hostnamev""::"0,"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible",","pid":level"4766:30}
,"time":"2023-09-22T23:21:36.24342261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 64733bc8-0e54-426a-b284-8ee7a1092e12, session_id: c328a90a-8f5c-4369-a069-ea688b82e263, gen: 10 } to UpstairsConnection { upstairs_id: 6d8e1874-8a0b-4075-9b57-a413f518a52b, session_id: 5b236919-eca9-476f-9f66-0439a7f4dbf6, gen: 11 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.243638742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Signaling to UpstairsConnection { upstairs_id: 64733bc8-0e54-426a-b284-8ee7a1092e12, session_id: c328a90a-8f5c-4369-a069-ea688b82e263, gen: 10 } thread that UpstairsConnection { upstairs_id: 6d8e1874-8a0b-4075-9b57-a413f518a52b, session_id: 5b236919-eca9-476f-9f66-0439a7f4dbf6, gen: 11 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.243691091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Crucible Downstairs promoting UpstairsConnection { upstairs_id: 6d8e1874-8a0b-4075-9b57-a413f518a52b, session_id: 5b236919-eca9-476f-9f66-0439a7f4dbf6, gen: 11 } to active, discarding 1 jobs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.243733373Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 6d8e1874-8a0b-4075-9b57-a413f518a52b, session_id: 5b236919-eca9-476f-9f66-0439a7f4dbf6, gen: 11 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.243771468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 64733bc8-0e54-426a-b284-8ee7a1092e12, session_id: c328a90a-8f5c-4369-a069-ea688b82e263, gen: 10 } cannot grab work lock, 64733bc8-0e54-426a-b284-8ee7a1092e12 is not active!","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.243810921Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_complete_work_cannot_see_none_different_upstairs_id ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.246891473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpPtzVoF/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.246945856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.248428839Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpICuwkB/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.248483193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.248519291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.248553238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: c4a9db90-cf08-4cd2-a281-2e3153e10dea","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.250289072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.250337487Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: debf9816-5036-4a29-971a-7959707ef7c4, session_id: cfd97f53-f668-4c56-a83f-a49fa1f1dd7b, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.250377753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: debf9816-5036-4a29-971a-7959707ef7c4, session_id: cfd97f53-f668-4c56-a83f-a49fa1f1dd7b, gen: 10 } to UpstairsConnection { upstairs_id: debf9816-5036-4a29-971a-7959707ef7c4, session_id: 05528500-99fe-4bae-be75-dfc22dbde9d4, gen: 11 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.25062074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Signaling to UpstairsConnection { upstairs_id: debf9816-5036-4a29-971a-7959707ef7c4, session_id: cfd97f53-f668-4c56-a83f-a49fa1f1dd7b, gen: 10 } thread that UpstairsConnection { upstairs_id: debf9816-5036-4a29-971a-7959707ef7c4, session_id: 05528500-99fe-4bae-be75-dfc22dbde9d4, gen: 11 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.250672994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Crucible Downstairs promoting UpstairsConnection { upstairs_id: debf9816-5036-4a29-971a-7959707ef7c4, session_id: 05528500-99fe-4bae-be75-dfc22dbde9d4, gen: 11 } to active, discarding 1 jobs","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.250715217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: debf9816-5036-4a29-971a-7959707ef7c4, session_id: 05528500-99fe-4bae-be75-dfc22dbde9d4, gen: 11 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.250753403Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: debf9816-5036-4a29-971a-7959707ef7c4, session_id: cfd97f53-f668-4c56-a83f-a49fa1f1dd7b, gen: 10 } cannot grab lock, does not match UpstairsConnection { upstairs_id: debf9816-5036-4a29-971a-7959707ef7c4, session_id: 05528500-99fe-4bae-be75-dfc22dbde9d4, gen: 11 }!","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.250796595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_complete_work_cannot_see_none_same_upstairs_id ... ok
{{"msg":""current number of open files limit 65536 is already the maximum"msg":","v":0,"current number of open files limit 65536 is already the maximumname":""crucible",","level"v"::300,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.254150803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",,""timepid""::"47662023-09-22T23:21:36.254156538Z"}
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4766"msg":"}
Opened existing region file \"/tmp/.tmpPtzVoF/region.json\"","v":{0,"name":""crucible"msg",:""level":30Created new region file \"/tmp/.tmp7h3aet/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.254220454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"}time":
"2023-09-22T23:21:36.254228573Z","{hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","Database read version 1"pid",:"4766v":0},
"name":"crucible","level":30,"time":"2023-09-22T23:21:36.254268461Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.254302867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: aa7fa17b-6d35-4d22-b991-2042801319b4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.261111124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.261162045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 9a57314d-b0f0-4fbd-bf04-117b6fe34801, session_id: bcd80027-a5a2-4e00-8334-f6e5f90f3646, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30Active Upstairs connections: [UpstairsConnection { upstairs_id: 9a57314d-b0f0-4fbd-bf04-117b6fe34801, session_id: bcd80027-a5a2-4e00-8334-f6e5f90f3646, gen: 10 }]
,"time":"2023-09-22T23:21:36.261851869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766Crucible Downstairs work queue:
}
DSW:[1000]  EClose  New deps:[]
DSW:[1001] EFClose  New deps:[]
DSW:[1002]  ReOpen  New deps:[JobId(1000)]
DSW:[1003]  ReOpen  New deps:[JobId(1001)]
Done tasks []
last_flush: JobId(0)
--------------------------------------
Got new work: [JobId(1000), JobId(1001), JobId(1002), JobId(1003)]
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.263384558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmp7h3aet/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.263437069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.263468845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.263502602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{test test::test_extent_new_close_flush_close ... "okmsg"
:"UUID: 37d7ca80-1c95-44f8-b547-ec9cebe4faaf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.267409026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.267454493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Before doing work we have:
Active Upstairs connections: [UpstairsConnection { upstairs_id: d247139f-5311-4ae1-914d-5f69c019b4de, session_id: b4d17432-7997-44b3-bf5b-8b1e5f7a26f7, gen: 10 }]
Crucible Downstairs work queue:
DSW:[1000]  EClose  New deps:[]
DSW:[1001] EFClose  New deps:[]
DSW:[1002]    Read  New deps:[JobId(1000), JobId(1001)]
DSW:[1003]    NoOp  New deps:[JobId(1000), JobId(1001), JobId(1002)]
DSW:[1004]  ReOpen  New deps:[JobId(1000), JobId(1001), JobId(1002), JobId(1003)]
Done tasks []
last_flush: JobId(0)
--------------------------------------
Got new work: [JobId(1000), JobId(1001), JobId(1002), JobId(1003), JobId(1004)]
Do IOop 1000
{"msg":"UpstairsConnection { upstairs_id: d247139f-5311-4ae1-914d-5f69c019b4de, session_id: b4d17432-7997-44b3-bf5b-8b1e5f7a26f7, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30{"msg",:""time":"2023-09-22T23:21:36.268199905Z"current number of open files limit 65536 is already the maximum,""hostname":","v":ip-10-150-1-55.us-west-2.compute.internal"0,","pid"name"::"4766crucible","}level"
:30,"time":"2023-09-22T23:21:36.268243809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpIBUDCt/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.268300572Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Got m: ExtentLiveCloseAck { upstairs_id: d247139f-5311-4ae1-914d-5f69c019b4de, session_id: b4d17432-7997-44b3-bf5b-8b1e5f7a26f7, job_id: JobId(1000), result: Ok((0, 0, false)) }
Do IOop 1001
Got m: ExtentLiveCloseAck { upstairs_id: d247139f-5311-4ae1-914d-5f69c019b4de, session_id: b4d17432-7997-44b3-bf5b-8b1e5f7a26f7, job_id: JobId(1001), result: Ok((0, 0, false)) }
Do IOop 1002
Got m: ReadResponse { upstairs_id: d247139f-5311-4ae1-914d-5f69c019b4de, session_id: b4d17432-7997-44b3-bf5b-8b1e5f7a26f7, job_id: JobId(1002), responses: Ok([ReadResponse { eid: 2, offset: Block { value: 1, shift: 9 }, data: b"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", block_contexts: [] }]) }
Do IOop 1003
Got m: ExtentLiveAckId { upstairs_id: d247139f-5311-4ae1-914d-5f69c019b4de, session_id: b4d17432-7997-44b3-bf5b-8b1e5f7a26f7, job_id: JobId(1003), result: Ok(()) }
Do IOop 1004
Got m: ExtentLiveAckId { upstairs_id: d247139f-5311-4ae1-914d-5f69c019b4de, session_id: b4d17432-7997-44b3-bf5b-8b1e5f7a26f7, job_id: JobId(1004), result: Ok(()) }
test test::test_extent_simple_close_flush_close ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.273641599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpfXGQZN/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.273779221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.274737821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpIBUDCt/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.274790702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.274827658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.274862526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: 0c6c1a78-03b9-4f93-b775-53b778c4926b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.27708078Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.277130314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Active Upstairs connections: [UpstairsConnection { upstairs_id: 8f3031f5-83bf-4e47-9bd4-4c6a88b9030a, session_id: 2b6beb2e-82c4-45d6-8df8-ab94de799177, gen: 10 }]
Crucible Downstairs work queue:
DSW:[1000]   Write  New deps:[]
DSW:[1001]  EClose  New deps:[JobId(1000)]
Done tasks []
last_flush: JobId(0)
--------------------------------------
Got new work: [JobId(1000), JobId(1001)]
{"msg":"UpstairsConnection { upstairs_id: 8f3031f5-83bf-4e47-9bd4-4c6a88b9030a, session_id: 2b6beb2e-82c4-45d6-8df8-ab94de799177, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.278317907Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.280770223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpfXGQZN/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.280826048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.28086314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.28089775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_extent_write_close ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.282173823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpWqEyfg/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.282255116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: a3036cd6-f3d8-4f1b-852c-75a6ecb2f8cf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.283056696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.283104786Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Active Upstairs connections: [UpstairsConnection { upstairs_id: aa96e365-81ae-4658-ba33-355ec5724697, session_id: 92f7a2cb-754e-4338-852f-d402d37e960c, gen: 10 }]
{"msg":"Crucible Downstairs work queue:
UpstairsConnection { upstairs_id: aa96e365-81ae-4658-ba33-355ec5724697, session_id: 92f7a2cb-754e-4338-852f-d402d37e960c, gen: 10 } is now active (read-write)"DSW:[1000]   Write  New deps:[]
,"v"DSW:[1001] EFClose  New deps:[JobId(1000)]
:Done tasks []
0last_flush: JobId(0)
,--------------------------------------
"name":"crucible","level":30Got new work: [JobId(1000), JobId(1001)]
,"time":"2023-09-22T23:21:36.283648143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_extent_write_flush_close ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.288621009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp4ll9pw/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.288788221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.289047625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpWqEyfg/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.289106681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.289145108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.289181229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: 97737660-1a46-4ea8-ab7d-8ff38e4e9cd9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.291741835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.291789205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 51fd8704-7826-4cb9-8726-3803fc9d81db, session_id: 8b3cee1a-cd10-48a6-ad9b-ec1df5477e78, gen: 10 } is now active (read-write)Active Upstairs connections: [UpstairsConnection { upstairs_id: 51fd8704-7826-4cb9-8726-3803fc9d81db, session_id: 8b3cee1a-cd10-48a6-ad9b-ec1df5477e78, gen: 10 }]
","v":0,"name":"crucible","level"Crucible Downstairs work queue:
:30DSW:[1000]   Write  New deps:[]
DSW:[1001]   Flush  New deps:[]
DSW:[1002]   Write  New deps:[JobId(1000), JobId(1001)]
,"DSW:[1003]  EClose  New deps:[JobId(1000), JobId(1001), JobId(1002)]
time":Done tasks []
"last_flush: JobId(0)
2023-09-22T23:21:36.292350278Z"--------------------------------------
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Got new work: [JobId(1000), JobId(1001), JobId(1002), JobId(1003)]
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.294541117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmp4ll9pw/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.294579262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.294599716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.294618607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_extent_write_flush_write_close ... ok
{"msg":"UUID: a4c27b10-4243-40e6-97b2-683044208503","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.297117706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.29716785Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.297479616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpSwLMyT/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.297548848Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 7c391f6f-c198-4917-aa35-1c75063f9878, session_id: 1635c1bd-824d-4dd9-9a89-8df7a5974793, gen: 10 } is now active (read-write)"Active Upstairs connections: [UpstairsConnection { upstairs_id: 7c391f6f-c198-4917-aa35-1c75063f9878, session_id: 1635c1bd-824d-4dd9-9a89-8df7a5974793, gen: 10 }]
,"v":0,"name":"crucible","level":30Crucible Downstairs work queue:
DSW:[1000]   Write  New deps:[]
DSW:[1001]   Write  New deps:[]
DSW:[1002] EFClose  New deps:[JobId(1000)]
DSW:[1003]  EClose  New deps:[JobId(1001)]
,Done tasks []
"last_flush: JobId(0)
time":--------------------------------------
"2023-09-22T23:21:36.29765977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Got new work: [JobId(1000), JobId(1001), JobId(1002), JobId(1003)]
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.301808064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpSwLMyT/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.301861209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.301901899Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.301934958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_extent_write_write_flush_close ... ok
{"msg":"UUID: bff87367-9361-47d4-a673-7bded39b1521","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.302703169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.302745012Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.302819215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpeno1fe/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.302879995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_multiple_read_only_no_job_id_collision ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.305250203Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp0Wl0vI/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.305301941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.306742222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpeno1fe/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.306789208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.306810574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.306836842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: e8fcc6b0-1f98-4bfd-ab37-79f94974d36d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.307826491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.307872149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.308830736Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmp0Wl0vI/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.308886564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.308932686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.308958625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_promote_to_active_multi_read_only_different_uuid ... ok
{"msg":"UUID: 3b8ecbef-099a-4919-b60a-4233ef43e424","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.309943382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.30998924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"{"msg":"current number of open files limit 65536 is already the maximum","v":Signaling to UpstairsConnection { upstairs_id: 6f6956d1-741a-416d-92f2-d3763223d0a0, session_id: 878abda3-5c53-4b83-a225-52cc439c2d50, gen: 1 } thread that UpstairsConnection { upstairs_id: 6f6956d1-741a-416d-92f2-d3763223d0a0, session_id: 5948108d-46c6-4e4a-8532-1ee9bc949c97, gen: 1 } is being promoted (read-only)"0,","name"v":":crucible0",","name"level":":crucible"30,"level":30,"time":,""time":"2023-09-22T23:21:36.310380396Z","2023-09-22T23:21:36.310378493Z"hostname":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4766,"pid"}:
4766}
{"msg":"Created new region file \"/tmp/.tmpq3Ts9a/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.310455472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_promote_to_active_multi_read_only_same_uuid ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.312109466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpb62GWu/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.312182945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.314286432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpq3Ts9a/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.314342448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.314384653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.314423014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: 9178ef01-f71a-4dbd-8623-68423516cf03","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.315333095Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.315382227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
ds1: MutexGuard { value: Downstairs { region: Region { dir: "/tmp/.tmpq3Ts9a", def: RegionDefinition { block_size: 512, extent_size: Block { value: 4, shift: 9 }, extent_count: 2, uuid: 9178ef01-f71a-4dbd-8623-68423516cf03, encrypted: false, database_read_version: 1, database_write_version: 1 }, extents: [Mutex { data: Opened(Extent { number: 0, read_only: false, block_size: 512, extent_size: Block { value: 4, shift: 9 }, iov_max: 1024, inner: Mutex { data: Inner { file: File { fd: 237 }, metadb: Connection { path: Some("/tmp/.tmpq3Ts9a/00/000/000.db") }, dirty_blocks: {} } } }) }, Mutex { data: Opened(Extent { number: 1, read_only: false, block_size: 512, extent_size: Block { value: 4, shift: 9 }, iov_max: 1024, inner: Mutex { data: Inner { file: File { fd: 241 }, metadb: Connection { path: Some("/tmp/.tmpq3Ts9a/00/000/001.db") }, dirty_blocks: {} } } }) }], dirty_extents: {}, read_only: false, log: Logger() }, lossy: false, read_errors: false, write_errors: false, flush_errors: false, active_upstairs: {}, dss: DsStatOuter { ds_stat_wrap: Mutex { is_locked: false, has_waiters: false } }, read_only: false, encrypted: false, address: None, repair_address: None, log: Logger() }, mutex: Mutex { is_locked: true, has_waiters: false } }

ds2: MutexGuard { value: Downstairs { region: Region { dir: "/tmp/.tmpq3Ts9a", def: RegionDefinition { block_size: 512, extent_size: Block { value: 4, shift: 9 }, extent_count: 2, uuid: 9178ef01-f71a-4dbd-8623-68423516cf03, encrypted: false, database_read_version: 1, database_write_version: 1 }, extents: [Mutex { data: Opened(Extent { number: 0, read_only: false, block_size: 512, extent_size: Block { value: 4, shift: 9 }, iov_max: 1024, inner: Mutex { data: Inner { file: File { fd: 237 }, metadb: Connection { path: Some("/tmp/.tmpq3Ts9a/00/000/000.db") }, dirty_blocks: {} } } }) }, Mutex { data: Opened(Extent { number: 1, read_only: false, block_size: 512, extent_size: Block { value: 4, shift: 9 }, iov_max: 1024, inner: Mutex { data: Inner { file: File { fd: 241 }, metadb: Connection { path: Some("/tmp/.tmpq3Ts9a/00/000/001.db") }, dirty_blocks: {} } } }) }], dirty_extents: {}, read_only: false, log: Logger() }, lossy: false, read_errors: false, write_errors: false, flush_errors: false, active_upstairs: {b76ea64e-5a32-4215-9476-0f488dbb3024: ActiveUpstairs { upstairs_connection: UpstairsConnection { upstairs_id: b76ea64e-5a32-4215-9476-0f488dbb3024, session_id: bf5fccf8-eef4-472f-8b71-fc1a31eaeb56, gen: 2 }, work: Mutex { is_locked: false, has_waiters: false }, terminate_sender: Sender { chan: Tx { inner: Chan { tx: Tx { block_tail: 0x5cc6a90, tail_position: 0 }, semaphore: Semaphore { semaphore: Semaphore { permits: 1 }, bound: 1 }, rx_waker: AtomicWaker, tx_count: 1, rx_fields: "..." } } } }}, dss: DsStatOuter { ds_stat_wrap: Mutex { is_locked: false, has_waiters: false } }, read_only: false, encrypted: false, address: None, repair_address: None, log: Logger() }, mutex: Mutex { is_locked: true, has_waiters: false } }

{{""msg":"msg":"current number of open files limit 65536 is already the maximum"UpstairsConnection { upstairs_id: b76ea64e-5a32-4215-9476-0f488dbb3024, session_id: bf5fccf8-eef4-472f-8b71-fc1a31eaeb56, gen: 2 } is now active (read-write)","v":,"0v",":name":"0crucible",","name":"level"crucible":,"level"30:30,",time"":"time":"2023-09-22T23:21:36.316131565Z"2023-09-22T23:21:36.316132874Z","hostname":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"ip-10-150-1-55.us-west-2.compute.internal":,"4766pid":4766}}

{"msg":"{"msg":"Opened existing region file \"/tmp/.tmpb62GWu/region.json\"Attempting RW takeover from UpstairsConnection { upstairs_id: b76ea64e-5a32-4215-9476-0f488dbb3024, session_id: bf5fccf8-eef4-472f-8b71-fc1a31eaeb56, gen: 2 } to UpstairsConnection { upstairs_id: 97418758-0f11-4547-a9ef-1a43890b320e, session_id: 32fbdd10-df06-43fe-9e8d-757ae10c99ee, gen: 1 }"",",v":"0v":,"0name",":"name"crucible":",crucible"","level":30level":40,"time":"2023-09-22T23:21:36.316249928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"2023-09-22T23:21:36.316263085Z"{,"hostname":""msg"ip-10-150-1-55.us-west-2.compute.internal":","pid":Database read version 1"4766,"v":}0
,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.316321294Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.316379558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_promote_to_active_multi_read_write_different_uuid_lower_gen ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.317693053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpGrq4Il/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.317838674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: 2e7ab951-802d-4955-bf93-556d497258b4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.31798861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.318045084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: ef548056-f8ca-406d-a2df-e9377f52dec3, session_id: b69f1e2d-5f26-44d8-8655-e0d1da715faf, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.318905446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: ef548056-f8ca-406d-a2df-e9377f52dec3, session_id: b69f1e2d-5f26-44d8-8655-e0d1da715faf, gen: 1 } to UpstairsConnection { upstairs_id: 47fd0e40-3a11-4b94-a18d-1eab3b74989b, session_id: 54d92ad8-4da7-4b3a-8148-02f158e67cb2, gen: 1 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.318977394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_promote_to_active_multi_read_write_different_uuid_same_gen ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.32087017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpeILeH1/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.320962486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Populated 10 extents by copying 51200 bytes (100 blocks)
Populated 10 extents by copying 51200 bytes (100 blocks)
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.321608995Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpGrq4Il/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.321665461Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.321706781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.321744206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: 2b71e199-4551-4f7e-a897-922324d9c9ed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.323629341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.323682571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.323955948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpeILeH1/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.324034583Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.324094675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.324149559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",Export total_size: 51200  Extent size:5120  Total Extents:10
"Exporting from start_block: 0  count:100
pid":4766}
Populated 10 extents by copying 51200 bytes (100 blocks)
{"msg":"UpstairsConnection { upstairs_id: 13f002c8-20b2-457c-9af0-0f9b5a05056c, session_id: 77629114-556d-464c-ac8e-9bd127fa69bc, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.324426048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 13f002c8-20b2-457c-9af0-0f9b5a05056c, session_id: 77629114-556d-464c-ac8e-9bd127fa69bc, gen: 1 } to UpstairsConnection { upstairs_id: 13f002c8-20b2-457c-9af0-0f9b5a05056c, session_id: 37e396c4-b9e3-49cb-9e72-4add2d50216e, gen: 2 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.324493685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Signaling to UpstairsConnection { upstairs_id: 13f002c8-20b2-457c-9af0-0f9b5a05056c, session_id: 77629114-556d-464c-ac8e-9bd127fa69bc, gen: 1 } thread that UpstairsConnection { upstairs_id: 13f002c8-20b2-457c-9af0-0f9b5a05056c, session_id: 37e396c4-b9e3-49cb-9e72-4add2d50216e, gen: 2 } is being promoted (read-write)","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.324544675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 13f002c8-20b2-457c-9af0-0f9b5a05056c, session_id: 37e396c4-b9e3-49cb-9e72-4add2d50216e, gen: 2 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.324586863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Populated 11 extents by copying 51712 bytes (101 blocks)
{"msg":"UUID: 9441f374-eacf-409a-80a9-79ff99067655","v":0,"name":"crucible","level":30test test::test_promote_to_active_multi_read_write_same_uuid_larger_gen ... ok
,"time":"2023-09-22T23:21:36.32529425Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.325346035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 15a5e302-b08f-4d85-a556-c77589e5d674, session_id: 06087ed2-beaf-4761-b7d9-adc46411651a, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.325812163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Attempting RW takeover from UpstairsConnection { upstairs_id: 15a5e302-b08f-4d85-a556-c77589e5d674, session_id: 06087ed2-beaf-4761-b7d9-adc46411651a, gen: 1 } to UpstairsConnection { upstairs_id: 15a5e302-b08f-4d85-a556-c77589e5d674, session_id: b1f64075-23f4-4c0a-8cdc-b0e5f86e9f7d, gen: 1 }","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.325872273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.325971705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpVuBd7H/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.326085981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_promote_to_active_multi_read_write_same_uuid_same_gen ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.327370558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpb88AZ2/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.327452519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Export total_size: 51200  Extent size:5120  Total Extents:10
Exporting from start_block: 0  count:100
Export total_size: 56320  Extent size:5120  Total Extents:11
Exporting from start_block: 0  count:101
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.329214844Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpVuBd7H/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.329271046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.329308508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.329351109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: 32f8c64d-d6f0-455d-a03e-b312f819d6a8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.330121711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.330172181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.330519781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpb88AZ2/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.330573123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.330607088Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.330627064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_promote_to_active_one_read_only ... ok
{"msg":"UUID: b57ff436-f771-4756-ac0c-c6bb6dd7b2b6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.331831926Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.331883166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.332019696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmphT0TAH/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.332098499Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 6fed7906-30ce-4ec1-bab3-26584d5634d0, session_id: 2d3f5bce-a17c-4a79-bbc6-f9cad82074ad, gen: 1 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.332243038Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_promote_to_active_one_read_write ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.333568548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmp6BWdqh/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.333649861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Read and wrote out 100 blocks
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.335771643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmphT0TAH/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.33583846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.335880947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.335916007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: fc483d0d-1ea7-4d46-a06e-c635454a441b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.337797691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":Active Upstairs connections: [UpstairsConnection { upstairs_id: 3b357141-7ef4-4ae1-8f8f-c6f0511eea53, session_id: 04a5e9c8-5fc7-455e-ba77-6099913b2300, gen: 10 }]
Crucible Downstairs work queue:
DSW:[1000]    Read  New deps:[]
DSW:[1001]    Read  New deps:[JobId(1000)]
Done tasks []
last_flush: JobId(0)
--------------------------------------
Got new work: [JobId(1000), JobId(1001)]
4766}
Do IOop 1000
{"msg":"Blocks per extent:4 Total Extents: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.338017585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UpstairsConnection { upstairs_id: 3b357141-7ef4-4ae1-8f8f-c6f0511eea53, session_id: 04a5e9c8-5fc7-455e-ba77-6099913b2300, gen: 10 } is now active (read-write)","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.338068308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Got m: ReadResponse { upstairs_id: 3b357141-7ef4-4ae1-8f8f-c6f0511eea53, session_id: 04a5e9c8-5fc7-455e-ba77-6099913b2300, job_id: JobId(1000), responses: Ok([ReadResponse { eid: 0, offset: Block { value: 1, shift: 9 }, data: b"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", block_contexts: [] }]) }
Do IOop 1001
Got m: ReadResponse { upstairs_id: 3b357141-7ef4-4ae1-8f8f-c6f0511eea53, session_id: 04a5e9c8-5fc7-455e-ba77-6099913b2300, job_id: JobId(1001), responses: Ok([ReadResponse { eid: 1, offset: Block { value: 1, shift: 9 }, data: b"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", block_contexts: [] }]) }
Active Upstairs connections: [UpstairsConnection { upstairs_id: 3b357141-7ef4-4ae1-8f8f-c6f0511eea53, session_id: 04a5e9c8-5fc7-455e-ba77-6099913b2300, gen: 10 }]
Crucible Downstairs work queue:  Empty
Done tasks [JobId(1000), JobId(1001)]
last_flush: JobId(0)
--------------------------------------
test test::import_test_too_small ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.338969238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpxwEdpG/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.339032896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Read and wrote out 101 blocks
test test::test_simple_read ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.340722506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpZt7Eys/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.340862366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.341372372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmp6BWdqh/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.341425134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.341463728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.34151627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
Read and wrote out 100 blocks
test test::import_test_basic_read_blocks ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.346041524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmppimBq2/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.346134071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"UUID: 81def555-b192-40a0-acf4-3ddea77e2c6d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.347138634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.347202669Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::import_test_too_large ... ok
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.348139335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Created new region file \"/tmp/.tmpo0lONP/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.348329502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ed48f294784d46ea7d4bb99336918b74358eca46\nCommit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main\nrustc: 1.70.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.348789707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.348858806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Using address: 127.0.0.1:5557","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.348902675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.349325934Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpZt7Eys/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.349392594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.349425097Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.349450862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{{"msg"":"msg":"current number of open files limit 65536 is already the maximum"Repair listens on 127.0.0.1:5558","v":,"0v,"":name":"0crucible,""name":,""crucible"level",":level":3030,"time":"2023-09-22T23:21:36.349615303Z",","hostname":time":""2023-09-22T23:21:36.349616017Z"ip-10-150-1-55.us-west-2.compute.internal",,""pid"hostname"::"4766}
ip-10-150-1-55.us-west-2.compute.internal","pid":4766{,"task":""repair"msg":"}
Opened existing region file \"/tmp/.tmpxwEdpG/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.349673042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.349708991Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.349743696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"listening","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.350680455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766test test::import_test_basic ... ok
,"local_addr":"127.0.0.1:5558","task":"repair"}
{"msg":"Using repair address: 127.0.0.1:5558","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.350919569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.350964169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"listening on 127.0.0.1:5557","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.351422135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"accepted connection from 127.0.0.1:52256","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.352314851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Connection request from 15649826-de3f-4a94-ae54-5732918c3786 with version 3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.352797992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"proc"}
{"msg":"UUID: 1b8d930c-af92-4946-b813-90e44a027831","v":{0,"name":"crucible","level":"30msg":"UUID: aad39680-8265-40f5-8f31-e66186f0fe36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.35305682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
,"time":"{2023-09-22T23:21:36.353068201Z",""hostname"msg:"":"ip-10-150-1-55.us-west-2.compute.internal"Blocks per extent:4 Total Extents: 5",",pid"":v":47660,"name":"}crucible"
,"level":30,"time":"2023-09-22T23:21:36.35311508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.35340481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.353477194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmppimBq2/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.353527495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.353562923Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.353595129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"{msg":""msg":"connection (127.0.0.1:52256) Exits with error: Required version 4, Or [3] got 3"Crucible Version: Crucible Version: 0.0.1\n,"v":0Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46\n,"name":"crucible"Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main\n,"level":50rustc: 1.70.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: true Opt level: 0","v":0,"name":"crucible","level":30,","time"time":":"2023-09-22T23:21:36.353659199Z"2023-09-22T23:21:36.353651304Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid"::47664766,"}task":
"main"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.353733146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Using address: 127.0.0.1:5555","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.3537666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Repair listens on 127.0.0.1:5556","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.353954357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"repair"}
{"msg":"listening","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.354098273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"local_addr":"127.0.0.1:5556","task":"repair"}
{"msg":"1002 job Read for connection UpstairsConnection { upstairs_id: e49f863f-8242-419c-9497-aa73fb0f3913, session_id: 8c10f889-cfe9-4164-a34e-713141a58437, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.354292961Z","hostname"{:"ip-10-150-1-55.us-west-2.compute.internal","pid"":msg4766":"}
Using repair address: 127.0.0.1:5556","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.354342009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.35439816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"listening on 127.0.0.1:5555","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.354608432Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ed48f294784d46ea7d4bb99336918b74358eca46\nCommit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main\nrustc: 1.70.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.354690041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.354736271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Using address: 127.0.0.1:5561","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.354777405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Repair listens on 127.0.0.1:5562","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.355037137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"repair"}
{{""msg":"msg"listening":","v":accepted connection from 127.0.0.1:52476"0,"name,"":v"":crucible"0,","level"name"::"30crucible","level":30,"time":"2023-09-22T23:21:36.355188321Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time,"":pid"":47662023-09-22T23:21:36.355191388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",,""pid":local_addr"4766:"127.0.0.1:5562,""task",":"task"main":"}repair"
}
{"msg":"Using repair address: 127.0.0.1:5562","v":{0,"name":"crucible","level":"30msg":"Connection request from acb09623-8b52-40d9-9ec9-f8e8a0e2acbd with version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.35540462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main","}time":
{"msg":"No SSL acceptor configured"","v":0,"2023-09-22T23:21:36.355416481Z"name":"crucible",",level"":hostname"30:"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"proc"}
,"time":"2023-09-22T23:21:36.355472077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{{"msg":"listening on 127.0.0.1:5561",""v":msg"0:","name":"crucible","level":30upstairs UpstairsConnection { upstairs_id: acb09623-8b52-40d9-9ec9-f8e8a0e2acbd, session_id: 88ebf34f-49b9-432d-8ca9-0cbead928022, gen: 1 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.355537457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
,"time":"2023-09-22T23:21:36.355549861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"proc"}
{"msg":"2002 job Flush for connection UpstairsConnection { upstairs_id: e49f863f-8242-419c-9497-aa73fb0f3913, session_id: 8c10f889-cfe9-4164-a34e-713141a58437, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.355724509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"accepted connection from 127.0.0.1:35336","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.35597939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Connection request from 7c6cc61c-e084-46c4-9a6b-2d866e33dfac with version 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.356289188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"proc"}
{"msg":"downstairs and upstairs using different but compatible versions, Upstairs is 5, but supports [4, 5], downstairs is 4","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.35635693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"proc"}
{"msg":"upstairs UpstairsConnection { upstairs_id: 7c6cc61c-e084-46c4-9a6b-2d866e33dfac, session_id: 1c566281-ee17-4f04-ac33-90bea0b26a2e, gen: 1 } connected, version 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.35647096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"proc"}
{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.356633853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Opened existing region file \"/tmp/.tmpo0lONP/region.json\"","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.35668167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.356720076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.356769393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_version_downrev ... ok
test test::two_job_chains ... ok
test test::test_version_match ... ok
test test::unblock_job ... ok
{"msg":"1002 job Read for connection UpstairsConnection { upstairs_id: d6889a6f-b791-4b23-96b2-7bad3c7074ce, session_id: 86f013ed-9310-4dfe-9691-063aa4bd272f, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.360073549Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::test_version_uprev_compatable ... ok
{"msg":"UUID: f0e1cc08-496b-4df1-af7b-29dd28d285c9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.36055191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.360600784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::unblock_job_chain ... ok
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ed48f294784d46ea7d4bb99336918b74358eca46\nCommit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main\nrustc: 1.70.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.361268682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.361328627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Using address: 127.0.0.1:5563","v":0,"name":"crucible","level":30test test::you_had_one_job ... ,ok"
time":"2023-09-22T23:21:36.361423871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"UUID: 7c42c1a4-22d9-45fd-a826-47736cc8907f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.361526057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Blocks per extent:4 Total Extents: 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.361568941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Repair listens on 127.0.0.1:5564","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.361621855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"repair"}
{"msg":"listening","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.361681201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"local_addr":"127.0.0.1:5564","task":"repair"}
{"msg":"1002 job Read for connection UpstairsConnection { upstairs_id: 0a96c1e0-3b8f-4a41-bdd0-0ac673cd2e7d, session_id: 013c3911-e491-4601-bc22-64918fd8155c, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40{"msg":"1002 job Read for connection UpstairsConnection { upstairs_id: b1435588-acb5-43f3-9d5b-5b81cde60797, session_id: f9e1212a-a737-4a66-baef-4fd64a6f829d, gen: 0 } waiting on 1 deps","v":,"0time":","name"2023-09-22T23:21:36.361809808Z":","crucible"hostname":","level"ip-10-150-1-55.us-west-2.compute.internal":,"pid":404766}
{"msg":"Using repair address: 127.0.0.1:5564","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.361841448Z",","time"hostname"::""2023-09-22T23:21:36.361903423Z"ip-10-150-1-55.us-west-2.compute.internal",","hostname"pid"::"4766ip-10-150-1-55.us-west-2.compute.internal"},"
pid":4766,"task":"main"}
{"msg":"No SSL acceptor configured","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.361965093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: ed48f294784d46ea7d4bb99336918b74358eca46\nCommit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main\nrustc: 1.70.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos  Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.362125714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":{0,"name":"crucible","level":"30msg":"listening on 127.0.0.1:5563","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.362176502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
,"time":"{2023-09-22T23:21:36.362187052Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal"Using address: 127.0.0.1:5579,"","pidv""::04766,"name":,""crucible"task",:""main"level":}30
,"time":"2023-09-22T23:21:36.362228048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"1002 job Flush for connection UpstairsConnection { upstairs_id: a6691f57-e53e-48bb-8d47-8d44e03a96a3, session_id: 58dea043-607b-4e68-9c88-b6528c30ae9e, gen: 0 } waiting on 1 deps","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.362355256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
{"msg":"Repair listens on 127.0.0.1:5560","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.362465745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"repair"}
test test::unblock_job_chain_second_is_flush ... {ok
"msg":"accepted connection from 127.0.0.1:37397","v":0,"{name":"crucible","level"":msg"30:"listening","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.362531869Z",","time":hostname":""2023-09-22T23:21:36.362538592Z"ip-10-150-1-55.us-west-2.compute.internal,"","hostname"pid:"":4766ip-10-150-1-55.us-west-2.compute.internal,"","taskpid""::4766"main"}
,"local_addr":"127.0.0.1:5560","task":"repair"}
{"msg":"Connection request from 5c71e2d7-67fc-4daf-b682-2fd653bbbf4e with version 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.362709086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"proc"}
{"msg":"{Using repair address: 127.0.0.1:5560","v":0,"name":""crucible"msg",":"level":30downstairs and upstairs using different but compatible versions, Upstairs is 8, but supports [3, 4, 5], downstairs is 4","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:36.362763698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":",No SSL acceptor configured"","timev""::"0,"2023-09-22T23:21:36.362773566Z"name":"crucible",","hostname"level":":30ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"proc"}
{"msg":","time":"2023-09-22T23:21:36.36291573Z","hostname":upstairs UpstairsConnection { upstairs_id: 5c71e2d7-67fc-4daf-b682-2fd653bbbf4e, session_id: 623344f4-c90d-4a6f-92d6-8f741bd9fc20, gen: 1 } connected, version 4"","v":ip-10-150-1-55.us-west-2.compute.internal"0,","pid"name"::4766"crucible",","task"level"::"30main"}
{"msg":"listening on 127.0.0.1:5579","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.362955893Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"proc"}
,"time":"2023-09-22T23:21:36.362974335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
test test::unblock_job_chain_first_is_flush ... ok
{"msg":"accepted connection from 127.0.0.1:40997","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.363254286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"main"}
{"msg":"Connection request from 77586f62-9e02-449a-b289-31377afff630 with version 5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:36.363600067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766,"task":"proc"}
{"msg":"connection (127.0.0.1:40997) Exits with error: Required version 4, Or [5] got 5","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:21:36.363662137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4766}
test test::unblock_job_upstairs_sends_big_deps ... ok
test test::test_version_uprev_only ... ok
test test::test_version_uprev_list ... ok
test region::test::test_flush_after_multiple_disjoint_writes ... ok
test region::test::test_big_extent_full_write_and_flush ... ok

test result: ok. 156 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.49s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_downstairs-3ed9735920c1592d --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_hammer-9622fb9be260fb45 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_integration_tests-8902d603847d3610 --nocapture`

running 57 tests
Sep 22 23:21:37.635 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.635 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.635 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.635 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.635 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.635 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.635 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.635 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.639 INFO Created new region file "/tmp/downstairs-hyFqsHgE/region.json"
Sep 22 23:21:37.639 INFO Created new region file "/tmp/downstairs-7jreVyVE/region.json"
Sep 22 23:21:37.639 INFO Created new region file "/tmp/downstairs-fNrmxQUi/region.json"
Sep 22 23:21:37.639 INFO Created new region file "/tmp/downstairs-tEDzCIA0/region.json"
Sep 22 23:21:37.639 INFO Created new region file "/tmp/downstairs-MOi2zaso/region.json"
Sep 22 23:21:37.639 INFO Created new region file "/tmp/downstairs-a8oaK2eo/region.json"
Sep 22 23:21:37.639 INFO Created new region file "/tmp/downstairs-qaZTfu42/region.json"
Sep 22 23:21:37.639 INFO Created new region file "/tmp/downstairs-kwQdCZxu/region.json"
Sep 22 23:21:37.652 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.652 INFO Opened existing region file "/tmp/downstairs-hyFqsHgE/region.json"
Sep 22 23:21:37.652 INFO Database read version 1
Sep 22 23:21:37.652 INFO Database write version 1
Sep 22 23:21:37.653 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.653 INFO Opened existing region file "/tmp/downstairs-fNrmxQUi/region.json"
Sep 22 23:21:37.653 INFO Database read version 1
Sep 22 23:21:37.653 INFO Database write version 1
Sep 22 23:21:37.654 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.654 INFO Opened existing region file "/tmp/downstairs-7jreVyVE/region.json"
Sep 22 23:21:37.654 INFO Database read version 1
Sep 22 23:21:37.654 INFO Database write version 1
Sep 22 23:21:37.655 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.655 INFO Opened existing region file "/tmp/downstairs-a8oaK2eo/region.json"
Sep 22 23:21:37.655 INFO Database read version 1
Sep 22 23:21:37.655 INFO Database write version 1
Sep 22 23:21:37.655 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.655 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.655 INFO Opened existing region file "/tmp/downstairs-kwQdCZxu/region.json"
Sep 22 23:21:37.655 INFO Opened existing region file "/tmp/downstairs-qaZTfu42/region.json"
Sep 22 23:21:37.655 INFO Database read version 1
Sep 22 23:21:37.655 INFO Database read version 1
Sep 22 23:21:37.655 INFO Database write version 1
Sep 22 23:21:37.655 INFO Database write version 1
Sep 22 23:21:37.655 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.655 INFO Opened existing region file "/tmp/downstairs-MOi2zaso/region.json"
Sep 22 23:21:37.655 INFO Database read version 1
Sep 22 23:21:37.655 INFO Database write version 1
Sep 22 23:21:37.656 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.656 INFO Opened existing region file "/tmp/downstairs-tEDzCIA0/region.json"
Sep 22 23:21:37.656 INFO Database read version 1
Sep 22 23:21:37.656 INFO Database write version 1
Sep 22 23:21:37.658 INFO UUID: f2d164d9-9cbb-47eb-a838-b7f17a4d0411
Sep 22 23:21:37.658 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.658 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.659 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.659 INFO Using address: 127.0.0.1:54884, task: main
Sep 22 23:21:37.661 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.661 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61071, task: repair
Sep 22 23:21:37.661 INFO UUID: ea1b6b61-6132-4ae7-8fec-05be62de606d
Sep 22 23:21:37.661 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.661 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61071, task: repair
Sep 22 23:21:37.661 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.661 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.661 INFO Using address: 127.0.0.1:39150, task: main
Sep 22 23:21:37.661 INFO listening, local_addr: 127.0.0.1:61071, task: repair
Sep 22 23:21:37.662 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.662 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50517, task: repair
Sep 22 23:21:37.662 INFO UUID: ca34832d-be06-44a7-9364-f6c3bc44d7fb
Sep 22 23:21:37.662 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.662 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50517, task: repair
Sep 22 23:21:37.662 INFO listening, local_addr: 127.0.0.1:50517, task: repair
Sep 22 23:21:37.662 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.662 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.662 INFO Using address: 127.0.0.1:46295, task: main
Sep 22 23:21:37.662 INFO UUID: f652e01a-1c15-4c9f-9d6f-06b9109bda2e
Sep 22 23:21:37.662 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.662 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.662 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.662 INFO Using address: 127.0.0.1:63497, task: main
Sep 22 23:21:37.662 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.663 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58361, task: repair
Sep 22 23:21:37.663 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58361, task: repair
Sep 22 23:21:37.663 INFO listening, local_addr: 127.0.0.1:58361, task: repair
Sep 22 23:21:37.663 INFO UUID: b778f7e5-519c-44d5-90ea-a92dbeffe813
Sep 22 23:21:37.663 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.663 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.663 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.663 INFO Using address: 127.0.0.1:36647, task: main
Sep 22 23:21:37.663 INFO UUID: a4638f9d-a209-4281-803d-046dfafa8f18
Sep 22 23:21:37.663 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.663 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.663 INFO UUID: ff162497-e5a3-4763-b129-29dec4869786
Sep 22 23:21:37.663 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.663 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.663 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.663 INFO Using address: 127.0.0.1:34554, task: main
Sep 22 23:21:37.663 INFO UUID: 2ad3fd00-b9f7-450b-935d-a7c44c290e9e
Sep 22 23:21:37.663 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.663 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.663 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.664 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.664 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.664 INFO Using address: 127.0.0.1:46967, task: main
Sep 22 23:21:37.664 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.664 INFO Using address: 127.0.0.1:43936, task: main
Sep 22 23:21:37.664 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39836, task: repair
Sep 22 23:21:37.664 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39836, task: repair
Sep 22 23:21:37.664 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46776, task: repair
Sep 22 23:21:37.664 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46776, task: repair
Sep 22 23:21:37.664 INFO listening, local_addr: 127.0.0.1:39836, task: repair
Sep 22 23:21:37.664 INFO listening, local_addr: 127.0.0.1:46776, task: repair
Sep 22 23:21:37.664 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.664 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:64111, task: repair
Sep 22 23:21:37.664 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.664 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:64111, task: repair
Sep 22 23:21:37.664 INFO listening, local_addr: 127.0.0.1:64111, task: repair
Sep 22 23:21:37.664 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.664 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38438, task: repair
Sep 22 23:21:37.664 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38438, task: repair
Sep 22 23:21:37.664 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51241, task: repair
Sep 22 23:21:37.664 INFO listening, local_addr: 127.0.0.1:38438, task: repair
Sep 22 23:21:37.664 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51241, task: repair
Sep 22 23:21:37.664 INFO listening, local_addr: 127.0.0.1:51241, task: repair
Sep 22 23:21:37.667 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51241, task: repair
Sep 22 23:21:37.667 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38438, task: repair
Sep 22 23:21:37.667 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46776, task: repair
Sep 22 23:21:37.667 INFO Using repair address: 127.0.0.1:51241, task: main
Sep 22 23:21:37.667 INFO Using repair address: 127.0.0.1:38438, task: main
Sep 22 23:21:37.667 INFO Using repair address: 127.0.0.1:46776, task: main
Sep 22 23:21:37.667 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.667 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.667 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.667 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58361, task: repair
Sep 22 23:21:37.667 INFO Using repair address: 127.0.0.1:58361, task: main
Sep 22 23:21:37.667 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.667 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61071, task: repair
Sep 22 23:21:37.667 INFO Using repair address: 127.0.0.1:61071, task: main
Sep 22 23:21:37.667 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.668 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50517, task: repair
Sep 22 23:21:37.668 INFO Using repair address: 127.0.0.1:50517, task: main
Sep 22 23:21:37.668 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:64111, task: repair
Sep 22 23:21:37.668 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.668 INFO Using repair address: 127.0.0.1:64111, task: main
Sep 22 23:21:37.668 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.668 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39836, task: repair
Sep 22 23:21:37.668 INFO Using repair address: 127.0.0.1:39836, task: main
Sep 22 23:21:37.668 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.668 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.668 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.668 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.668 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.668 INFO Created new region file "/tmp/downstairs-pWkbItfq/region.json"
Sep 22 23:21:37.668 INFO Created new region file "/tmp/downstairs-75rNZW4A/region.json"
Sep 22 23:21:37.668 INFO Created new region file "/tmp/downstairs-n0vNnrMi/region.json"
Sep 22 23:21:37.668 INFO Created new region file "/tmp/downstairs-Nlgr6SUH/region.json"
Sep 22 23:21:37.668 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.668 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.668 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.668 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.668 INFO Created new region file "/tmp/downstairs-k8ooJOk2/region.json"
Sep 22 23:21:37.668 INFO Created new region file "/tmp/downstairs-TZ4p57jk/region.json"
Sep 22 23:21:37.668 INFO Created new region file "/tmp/downstairs-jXVezaZ4/region.json"
Sep 22 23:21:37.668 INFO Created new region file "/tmp/downstairs-CZzWC74T/region.json"
Sep 22 23:21:37.672 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.672 INFO Opened existing region file "/tmp/downstairs-pWkbItfq/region.json"
Sep 22 23:21:37.672 INFO Database read version 1
Sep 22 23:21:37.672 INFO Database write version 1
Sep 22 23:21:37.673 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.673 INFO Opened existing region file "/tmp/downstairs-n0vNnrMi/region.json"
Sep 22 23:21:37.673 INFO Database read version 1
Sep 22 23:21:37.673 INFO Database write version 1
Sep 22 23:21:37.674 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.674 INFO Opened existing region file "/tmp/downstairs-Nlgr6SUH/region.json"
Sep 22 23:21:37.674 INFO Database read version 1
Sep 22 23:21:37.674 INFO Database write version 1
Sep 22 23:21:37.674 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.674 INFO Opened existing region file "/tmp/downstairs-k8ooJOk2/region.json"
Sep 22 23:21:37.674 INFO Database read version 1
Sep 22 23:21:37.674 INFO Database write version 1
Sep 22 23:21:37.674 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.674 INFO Opened existing region file "/tmp/downstairs-75rNZW4A/region.json"
Sep 22 23:21:37.674 INFO Database read version 1
Sep 22 23:21:37.674 INFO Database write version 1
Sep 22 23:21:37.675 INFO UUID: 4e7888c2-7beb-4f8a-a2f2-dd1592f0ed78
Sep 22 23:21:37.675 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.675 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.675 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.675 INFO Using address: 127.0.0.1:40751, task: main
Sep 22 23:21:37.675 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.676 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.676 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.676 INFO Opened existing region file "/tmp/downstairs-TZ4p57jk/region.json"
Sep 22 23:21:37.676 INFO Opened existing region file "/tmp/downstairs-jXVezaZ4/region.json"
Sep 22 23:21:37.676 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52716, task: repair
Sep 22 23:21:37.676 INFO Database read version 1
Sep 22 23:21:37.676 INFO Database read version 1
Sep 22 23:21:37.676 INFO Database write version 1
Sep 22 23:21:37.676 INFO Database write version 1
Sep 22 23:21:37.676 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52716, task: repair
Sep 22 23:21:37.676 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.676 INFO listening, local_addr: 127.0.0.1:52716, task: repair
Sep 22 23:21:37.676 INFO Opened existing region file "/tmp/downstairs-CZzWC74T/region.json"
Sep 22 23:21:37.676 INFO Database read version 1
Sep 22 23:21:37.676 INFO Database write version 1
Sep 22 23:21:37.676 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52716, task: repair
Sep 22 23:21:37.676 INFO Using repair address: 127.0.0.1:52716, task: main
Sep 22 23:21:37.676 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.676 INFO UUID: f7971d88-fcac-4a34-864e-16881c021a65
Sep 22 23:21:37.676 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.676 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.676 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.676 INFO Using address: 127.0.0.1:43385, task: main
Sep 22 23:21:37.677 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.677 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35126, task: repair
Sep 22 23:21:37.677 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35126, task: repair
Sep 22 23:21:37.677 INFO listening, local_addr: 127.0.0.1:35126, task: repair
Sep 22 23:21:37.677 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.677 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35126, task: repair
Sep 22 23:21:37.677 INFO Created new region file "/tmp/downstairs-Avl7H1Y2/region.json"
Sep 22 23:21:37.677 INFO Using repair address: 127.0.0.1:35126, task: main
Sep 22 23:21:37.677 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.678 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.679 INFO Created new region file "/tmp/downstairs-j0KkEAG7/region.json"
Sep 22 23:21:37.679 INFO UUID: d803d148-f26b-4671-a1b7-af69788d7c3f
Sep 22 23:21:37.679 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.679 INFO UUID: 4b784146-9bfd-4506-a702-f66a5880a2a3
Sep 22 23:21:37.679 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.679 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.679 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.679 INFO Using address: 127.0.0.1:55221, task: main
Sep 22 23:21:37.679 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.679 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.679 INFO Using address: 127.0.0.1:38174, task: main
Sep 22 23:21:37.679 INFO UUID: 51f6c6d4-74e6-40e9-8073-39006e8120ae
Sep 22 23:21:37.679 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.679 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.679 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.679 INFO Using address: 127.0.0.1:64794, task: main
Sep 22 23:21:37.679 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.679 INFO UUID: ab7c04fa-de73-4af6-8f95-d0dc6939a823
Sep 22 23:21:37.679 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.680 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.680 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.680 INFO Using address: 127.0.0.1:39320, task: main
Sep 22 23:21:37.680 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44687, task: repair
Sep 22 23:21:37.679 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.680 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.680 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44687, task: repair
Sep 22 23:21:37.680 INFO listening, local_addr: 127.0.0.1:44687, task: repair
Sep 22 23:21:37.680 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52116, task: repair
Sep 22 23:21:37.680 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:59717, task: repair
Sep 22 23:21:37.680 INFO UUID: f9162fd6-864f-4c42-be11-73fe673d33ac
Sep 22 23:21:37.680 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52116, task: repair
Sep 22 23:21:37.680 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.680 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:59717, task: repair
Sep 22 23:21:37.680 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.680 INFO listening, local_addr: 127.0.0.1:52116, task: repair
Sep 22 23:21:37.680 INFO listening, local_addr: 127.0.0.1:59717, task: repair
Sep 22 23:21:37.680 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.680 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44687, task: repair
Sep 22 23:21:37.680 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.680 INFO Using address: 127.0.0.1:59903, task: main
Sep 22 23:21:37.680 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35012, task: repair
Sep 22 23:21:37.680 INFO Using repair address: 127.0.0.1:44687, task: main
Sep 22 23:21:37.680 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.680 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35012, task: repair
Sep 22 23:21:37.680 INFO listening, local_addr: 127.0.0.1:35012, task: repair
Sep 22 23:21:37.680 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52116, task: repair
Sep 22 23:21:37.680 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:59717, task: repair
Sep 22 23:21:37.680 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35012, task: repair
Sep 22 23:21:37.680 INFO Using repair address: 127.0.0.1:59717, task: main
Sep 22 23:21:37.680 INFO Using repair address: 127.0.0.1:35012, task: main
Sep 22 23:21:37.680 INFO UUID: edb26b00-76c7-4c82-b262-a6cf7181b4aa
Sep 22 23:21:37.680 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.680 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.680 INFO Using repair address: 127.0.0.1:52116, task: main
Sep 22 23:21:37.680 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.680 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.681 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.681 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.681 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.681 INFO Using address: 127.0.0.1:62341, task: main
Sep 22 23:21:37.681 INFO Created new region file "/tmp/downstairs-3KVbZFEh/region.json"
Sep 22 23:21:37.681 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.681 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:45979, task: repair
Sep 22 23:21:37.681 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:45979, task: repair
Sep 22 23:21:37.681 INFO listening, local_addr: 127.0.0.1:45979, task: repair
Sep 22 23:21:37.681 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.681 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58643, task: repair
Sep 22 23:21:37.681 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:45979, task: repair
Sep 22 23:21:37.681 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58643, task: repair
Sep 22 23:21:37.681 INFO Using repair address: 127.0.0.1:45979, task: main
Sep 22 23:21:37.681 INFO listening, local_addr: 127.0.0.1:58643, task: repair
Sep 22 23:21:37.681 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.681 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.681 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.681 INFO Created new region file "/tmp/downstairs-ClNSXxBw/region.json"
Sep 22 23:21:37.681 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58643, task: repair
Sep 22 23:21:37.681 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.681 INFO Created new region file "/tmp/downstairs-XyohQkMW/region.json"
Sep 22 23:21:37.681 INFO Using repair address: 127.0.0.1:58643, task: main
Sep 22 23:21:37.681 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.681 INFO Created new region file "/tmp/downstairs-gG7qY3T8/region.json"
Sep 22 23:21:37.682 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.682 INFO Created new region file "/tmp/downstairs-wQIcFXNr/region.json"
Sep 22 23:21:37.682 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.682 INFO Created new region file "/tmp/downstairs-olr9HIrf/region.json"
Sep 22 23:21:37.682 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.682 INFO Opened existing region file "/tmp/downstairs-Avl7H1Y2/region.json"
Sep 22 23:21:37.682 INFO Database read version 1
Sep 22 23:21:37.682 INFO Database write version 1
Sep 22 23:21:37.684 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.684 INFO Opened existing region file "/tmp/downstairs-j0KkEAG7/region.json"
Sep 22 23:21:37.684 INFO Database read version 1
Sep 22 23:21:37.684 INFO Database write version 1
Sep 22 23:21:37.685 INFO UUID: 7bf7d359-28ba-4346-a3ee-541f5bc5a1c0
Sep 22 23:21:37.685 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.686 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.686 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.686 INFO Using address: 127.0.0.1:42674, task: main
Sep 22 23:21:37.686 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.686 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33608, task: repair
Sep 22 23:21:37.686 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33608, task: repair
Sep 22 23:21:37.686 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.686 INFO Opened existing region file "/tmp/downstairs-3KVbZFEh/region.json"
Sep 22 23:21:37.686 INFO Database read version 1
Sep 22 23:21:37.686 INFO listening, local_addr: 127.0.0.1:33608, task: repair
Sep 22 23:21:37.686 INFO Database write version 1
Sep 22 23:21:37.686 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33608, task: repair
Sep 22 23:21:37.686 INFO Using repair address: 127.0.0.1:33608, task: main
Sep 22 23:21:37.686 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.687 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.687 INFO Opened existing region file "/tmp/downstairs-XyohQkMW/region.json"
Sep 22 23:21:37.687 INFO Database read version 1
Sep 22 23:21:37.687 INFO Database write version 1
Sep 22 23:21:37.688 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.688 INFO Opened existing region file "/tmp/downstairs-ClNSXxBw/region.json"
Sep 22 23:21:37.688 INFO Database read version 1
Sep 22 23:21:37.688 INFO Database write version 1
Sep 22 23:21:37.688 INFO UUID: 4afe10fd-86f8-4da0-8ac1-1f760ded452a
Sep 22 23:21:37.689 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.689 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.689 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.689 INFO Using address: 127.0.0.1:64157, task: main
Sep 22 23:21:37.689 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.689 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:37.689 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.689 INFO Opened existing region file "/tmp/downstairs-gG7qY3T8/region.json"
Sep 22 23:21:37.689 INFO Database read version 1
Sep 22 23:21:37.689 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:37.689 INFO Database write version 1
Sep 22 23:21:37.689 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.689 INFO listening, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:37.689 INFO Opened existing region file "/tmp/downstairs-wQIcFXNr/region.json"
Sep 22 23:21:37.689 INFO Database read version 1
Sep 22 23:21:37.689 INFO Database write version 1
Sep 22 23:21:37.689 INFO UUID: 74dd6474-4427-416d-8d80-1f3912278411
Sep 22 23:21:37.689 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.689 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:37.689 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.689 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.689 INFO Using repair address: 127.0.0.1:39249, task: main
Sep 22 23:21:37.689 INFO Using address: 127.0.0.1:59938, task: main
Sep 22 23:21:37.689 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.690 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.690 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50240, task: repair
Sep 22 23:21:37.690 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50240, task: repair
Sep 22 23:21:37.690 INFO listening, local_addr: 127.0.0.1:50240, task: repair
Sep 22 23:21:37.690 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50240, task: repair
Sep 22 23:21:37.690 INFO Using repair address: 127.0.0.1:50240, task: main
Sep 22 23:21:37.690 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.691 INFO Upstairs starts
Sep 22 23:21:37.691 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.691 INFO Upstairs <-> Downstairs Message Version: 4
{"msg":"Upstairs starts","v":0,"name":"crucible","level":30Sep 22 23:21:37.691 INFO Crucible stats registered with UUID: f712941d-0577-43e4-8a2e-7814ec270c09
Sep 22 23:21:37.691 INFO Crucible f712941d-0577-43e4-8a2e-7814ec270c09 has session id: 2db8c46a-7d80-4db6-a4af-05be116bdc65
,"time":"2023-09-22T23:21:37.69144035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\nSep 22 23:21:37.691 INFO UUID: 440ce80c-c0c9-48b9-ba41-98710683ab87
    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumosSep 22 23:21:37.691 INFO Blocks per extent:5 Total Extents: 2
\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30Sep 22 23:21:37.692 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.692 INFO Upstairs <-> Downstairs Message Version: 4, task: main
,"time":"2023-09-22T23:21:37.691994616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769Sep 22 23:21:37.692 INFO Using address: 127.0.0.1:57826, task: main
}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.692126275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{{"msg":"Upstairs starts","v":0,"name":"crucible","level":30"msg":"Crucible stats registered with UUID: 8e88d9c1-84f3-4981-affe-0c1310d04226",,""v"time"::"0,"name2023-09-22T23:21:37.692513464Z"":"crucible,"",hostname"":"level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":","Crucible Version: BuildInfo {time"\n:"    version: \"0.0.12023-09-22T23:21:37.692549962Z\"",\n,"    git_sha: \"hostname":"ed48f294784d46ea7d4bb99336918b74358eca46\",\nip-10-150-1-55.us-west-2.compute.internal","    git_commit_timestamp: pid\"":47692023-09-22T22:51:18.000000000Z\",\n}
    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n{    rustc_channel: \"stable\",\n"    rustc_host_triple: \"msg":"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"Crucible 8e88d9c1-84f3-4981-affe-0c1310d04226 has session id: 8e7645e9-bbcd-4dae-b03b-42e001b696e390c541806f23a127002de5b4038be731ba1458ca"\",\n,"v":    cargo_triple: \"0,"namex86_64-unknown-illumos"\":",\ncrucible"    debug: true,,\n"level"    opt_level: 0,:\n30}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.692601839Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:21:37.69260832Zpid"":,4769"hostname}"
:"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.69265253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible stats registered with UUID: 1c48f237-34b8-4484-b38b-7c6b80300cc8","v":0,"name":"crucible","level":30,"time":"Sep 22 23:21:37.692 INFO Repair listens on 127.0.0.1:0, task: repair
2023-09-22T23:21:37.692687614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible 1c48f237-34b8-4484-b38b-7c6b80300cc8 has session id: a5c3fa2b-8808-4f8f-9c13-31707eb7c936","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.692729992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.692 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44890, task: repair
Sep 22 23:21:37.692 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44890, task: repair
Sep 22 23:21:37.692 INFO listening, local_addr: 127.0.0.1:44890, task: repair
Sep 22 23:21:37.693 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44890, task: repair
Sep 22 23:21:37.693 INFO Using repair address: 127.0.0.1:44890, task: main
Sep 22 23:21:37.693 INFO No SSL acceptor configured, task: main
The guest has requested activation
The guest has requested activation
The guest has requested activation
Sep 22 23:21:37.693 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.693 INFO Opened existing region file "/tmp/downstairs-olr9HIrf/region.json"
Sep 22 23:21:37.693 INFO Database read version 1
Sep 22 23:21:37.693 INFO Database write version 1
Sep 22 23:21:37.694 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.694 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.694 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.694 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.694 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.694 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.694 INFO UUID: 61a97a70-da7b-4cb5-91fa-9da487068e7c
Sep 22 23:21:37.694 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.694 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.694 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.694 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.694 INFO UUID: 10863a73-c1c6-434a-bb7b-15af38914b1b
Sep 22 23:21:37.694 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.694 INFO UUID: 88162c01-9da1-4e8d-8218-aee73fa16e70
Sep 22 23:21:37.694 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.694 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.694 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.694 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.694 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.694 INFO Using address: 127.0.0.1:43168, task: main
Sep 22 23:21:37.694 INFO Using address: 127.0.0.1:34655, task: main
Sep 22 23:21:37.694 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.694 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.694 INFO Using address: 127.0.0.1:36502, task: main
Sep 22 23:21:37.694 INFO [0] connecting to 127.0.0.1:46295, looper: 0
{"msg":"Upstairs starts"{,"v":0,"name":""cruciblemsg"":","level":[0] connecting to 127.0.0.1:36647"30,"v":0,"name":"crucible","level":30{"msg":"[0] connecting to 127.0.0.1:46967","v":0,"name":"crucible,"","time"level":,:""30time":"2023-09-22T23:21:37.69489497Z"2023-09-22T23:21:37.694887787Z","hostname":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pidip-10-150-1-55.us-west-2.compute.internal"":,4769"pid":,4769"looper":"0"}
}
,{"time":"2023-09-22T23:21:37.694922109Z""msg",:""hostname":"Crucible Version: BuildInfo {\nip-10-150-1-55.us-west-2.compute.internal"    version: \","0.0.1\"pid",\n:4769    git_sha: \","ed48f294784d46ea7d4bb99336918b74358eca46\"looper",\n:"0"    git_commit_timestamp: \"}
2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.695003069Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.695051367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible stats registered with UUID: 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.695088594Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d has session id: 267e8ce7-7645-4600-9b29-59a06b3738f2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.695125847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.695 INFO [1] connecting to 127.0.0.1:43385, looper: 1
{"msg":"[1] connecting to 127.0.0.1:55221","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.695308885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1"}
Sep 22 23:21:37.695 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.695 INFO [2] connecting to 127.0.0.1:64157, looper: 2
Sep 22 23:21:37.695 INFO Repair listens on 127.0.0.1:0, task: repair
{"msg":"[1] connecting to 127.0.0.1:40751","v":0,"name":"crucible","level":30{"msg":"[2] connecting to 127.0.0.1:59938","v":0,"name":"crucible",,""timelevel""::"30The guest has requested activation
2023-09-22T23:21:37.695446374Z"Sep 22 23:21:37.695 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46369, task: repair
Sep 22 23:21:37.695 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.695 INFO listening on 127.0.0.1:0, task: main
,"time":"2023-09-22T23:21:37.69547072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2"}
Sep 22 23:21:37.695 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.695 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46369, task: repair
Sep 22 23:21:37.695 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.695 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.695 INFO Flush timeout: 0.5
Sep 22 23:21:37.695 INFO listening, local_addr: 127.0.0.1:46369, task: repair
,Sep 22 23:21:37.695 INFO listening on 127.0.0.1:0, task: main
"hostname":{"ip-10-150-1-55.us-west-2.compute.internal",""pidmsg""::"4769up_listen starts",,""v"looper:":"01","}name"
:"crucible","level":30{"msg":"[2] connecting to 127.0.0.1:42674","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.695822504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",","time"pid:"":47692023-09-22T23:21:37.69583393Z",,""hostnametask""::""up_listen"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper{":"2"}
"msg":"{Wait for all three downstairs to come online",""msgv""::"0,"up_listen starts"name":,""vcrucible"":0,","level"name:":30"crucible","level":30,,""timetime""::""2023-09-22T23:21:37.695882977Z2023-09-22T23:21:37.69588068Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47694769},
"Sep 22 23:21:37.695 INFO f712941d-0577-43e4-8a2e-7814ec270c09 active request set
Sep 22 23:21:37.695 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56492, task: repair
{task":"up_listen""}
msg":"Flush timeout: 0.5{","v"":msg0":,""name":"crucible"Wait for all three downstairs to come online,"",level""v:":300,"name":"crucible","level":30,"time":","time2023-09-22T23:21:37.695926987Z"":","hostname"2023-09-22T23:21:37.695930483Z:"",Sep 22 23:21:37.695 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46369, task: repair
"ip-10-150-1-55.us-west-2.compute.internal"hostname",:""Sep 22 23:21:37.695 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56492, task: repair
pidip-10-150-1-55.us-west-2.compute.internal"":,4769"}pid"
:4769}
{"msg":"Flush timeout: 0.5","v":0,"name":"{crucible",""level"msg:":30"1c48f237-34b8-4484-b38b-7c6b80300cc8 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.695975787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",","time"pid:"":4769}
2023-09-22T23:21:37.695983574Z"Sep 22 23:21:37.695 INFO Using repair address: 127.0.0.1:46369, task: main
{Sep 22 23:21:37.695 INFO listening, local_addr: 127.0.0.1:56492, task: repair
,""msg"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"8e88d9c1-84f3-4981-affe-0c1310d04226 active request set",","pid"v:":47690,"}
name":"crucible","level":30Sep 22 23:21:37.696 INFO No SSL acceptor configured, task: main
,"time":"2023-09-22T23:21:37.696019167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] connecting to 127.0.0.1:43936","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.696091865Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
{"msg":"[1] connecting to 127.0.0.1:39320","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.696133172Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1"}
{"msg":"[2] connecting to 127.0.0.1:57826","v":0,"name":"crucible","level":30Sep 22 23:21:37.696 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56492, task: repair
,"time":"2023-09-22T23:21:37.696169026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2"}
{"msg":"up_listen starts","v":0,"name":"crucible","level":30Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:37490, task: main
,"time":"2023-09-22T23:21:37.69621327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:49080, task: main
,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.696270029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30Sep 22 23:21:37.696 INFO Using repair address: 127.0.0.1:56492, task: main
,"time":"2023-09-22T23:21:37.696309132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"Sep 22 23:21:37.696 INFO No SSL acceptor configured, task: main
msg":"69c4c5e1-2fd5-4777-bf6a-fcd75618f55d active request set","v":0,"name":"crucible","level":30Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:40329, task: main
,"time":"Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:59058, task: main
Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:40703, task: main
Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:45668, task: main
2023-09-22T23:21:37.696357793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.696 INFO UUID: e80e1f8b-7f5a-43fe-b856-a2bd94bd70c3
Sep 22 23:21:37.696 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:57639, task: main
Sep 22 23:21:37.696 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35046, task: repair
Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:49558, task: main
Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:40898, task: main
Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:40210, task: main
Sep 22 23:21:37.696 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.696 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35046, task: repair
Sep 22 23:21:37.696 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.696 INFO Using address: 127.0.0.1:51713, task: main
Sep 22 23:21:37.696 INFO accepted connection from 127.0.0.1:60563, task: main
{"msg":"[0] 1c48f237-34b8-4484-b38b-7c6b80300cc8 looper connected","v":0,"name":"crucible"Sep 22 23:21:37.696 INFO [0] f712941d-0577-43e4-8a2e-7814ec270c09 looper connected, looper: 0
,"level":30Sep 22 23:21:37.696 INFO listening, local_addr: 127.0.0.1:35046, task: repair
,"time":"2023-09-22T23:21:37.696776987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
{"msg":"[0] 8e88d9c1-84f3-4981-affe-0c1310d04226 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.696870353Z","{hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":"4769msg":,""looper":"0"}[0] Proc runs for 127.0.0.1:46967 in state New"
,"v":0,"name":"crucible"{,"level":30"msg":"[0] Proc runs for 127.0.0.1:36647 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.696917047Z","hostname":"Sep 22 23:21:37.696 INFO [0] Proc runs for 127.0.0.1:46295 in state New
ip-10-150-1-55.us-west-2.compute.internal",,Sep 22 23:21:37.696 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35046, task: repair
""pid":time4769":"}2023-09-22T23:21:37.696926118Z"
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d looper connected"Sep 22 23:21:37.696 INFO Using repair address: 127.0.0.1:35046, task: main
,"v":0,"name":"crucible","level":30Sep 22 23:21:37.697 INFO No SSL acceptor configured, task: main
,"time":"2023-09-22T23:21:37.697006965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
Sep 22 23:21:37.697 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.697 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42362, task: repair
{"msg":"Upstairs starts","v":0,"name":"crucible"Sep 22 23:21:37.697 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42362, task: repair
,"level":30,"time":"2023-09-22T23:21:37.697243772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n{    git_branch: \"main\",\n    rustc_semver: \""1.70.0\"msg,"\n:"    rustc_channel: \"stable\",\n[1] 1c48f237-34b8-4484-b38b-7c6b80300cc8 looper connected"    rustc_host_triple: \","vx86_64-unknown-illumos\"",:\n0,    rustc_commit_sha: "\"name":"crucible"90c541806f23a127002de5b4038be731ba1458ca\",",\nlevel":    cargo_triple: \"30x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}"Sep 22 23:21:37.697 INFO [1] f712941d-0577-43e4-8a2e-7814ec270c09 looper connected, looper: 1
,{"v":0,"name":"crucible","level"":30msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.69734695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{,,"""msgtime""::""time":"2023-09-22T23:21:37.697322444ZUpstairs <-> Downstairs Message Version: 4""2023-09-22T23:21:37.697356779Z",,""v"hostname"::"0,","hostname"ip-10-150-1-55.us-west-2.compute.internalname"",:""Sep 22 23:21:37.697 INFO [1] Proc runs for 127.0.0.1:43385 in state New
:"cruciblepid"":,4769"level"ip-10-150-1-55.us-west-2.compute.internal":,,30""looper"pid":":1"4769}
}
{,"time":""msg"2023-09-22T23:21:37.697409168Z:"","{hostname":[1] Proc runs for 127.0.0.1:55221 in state New"","v":0ip-10-150-1-55.us-west-2.compute.internal,""",name"":pid"":crucible4769"msg},
""level"{::{"30"Crucible Version: BuildInfo {msg""\n:msg    version: ""\":0.0.1"\",\nCrucible stats registered with UUID: 8f69534d-528b-4c23-b2c9-ce2dfe832ae1"    git_sha: \",[1] 8e88d9c1-84f3-4981-affe-0c1310d04226 looper connected",""v,ed48f294784d46ea7d4bb99336918b74358eca46\""time""v::0",,""name\n"2023-09-22T23:21:37.697442914Z:"":crucible,"    git_commit_timestamp: ,""\"0hostnamelevel""::"302023-09-22T22:51:18.000000000Z\",,\n"ip-10-150-1-55.us-west-2.compute.internal"    git_branch: name"\"Sep 22 23:21:37.697 INFO listening, local_addr: 127.0.0.1:42362, task: repair
:,,Sep 22 23:21:37.697 INFO [2] f712941d-0577-43e4-8a2e-7814ec270c09 looper connected, looper: 2
main\"""pid""timecrucible":":4769",}\n
2023-09-22T23:21:37.697473536Z,""    rustc_semver: ,\"level1.70.0\"",:"\n30hostname":"    rustc_channel: \"stable\",\nip-10-150-1-55.us-west-2.compute.internal","    rustc_host_triple: \"pid":4769x86_64-unknown-illumos\"},
\n    rustc_commit_sha: \"{90c541806f23a127002de5b4038be731ba1458ca\",\n"msg"    cargo_triple: :\""x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\nCrucible 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 has session id: fdc9bcf8-a539-475f-a4b2-6afeca56855a}",",""v"time"::,""0v",":name"2023-09-22T23:21:37.697514944Z0":","crucible","name"hostname:","":crucible"level","":levelip-10-150-1-55.us-west-2.compute.internal"",:"3030pid":4769,"looper":"1"}
,"time":"2023-09-22T23:21:37.697562375Z","hostname":",ip-10-150-1-55.us-west-2.compute.internal"","timepid""::4769"}
Sep 22 23:21:37.697 INFO [2] Proc runs for 127.0.0.1:64157 in state New
2023-09-22T23:21:37.69756306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.697628625Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{Sep 22 23:21:37.697 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42362, task: repair
"msg":"Crucible stats registered with UUID: 82efa4d5-f84d-4765-a042-3cbb8c544041","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.697677994Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769Sep 22 23:21:37.697 INFO Using repair address: 127.0.0.1:42362, task: main
}
{"msg":"Sep 22 23:21:37.697 INFO No SSL acceptor configured, task: main
Crucible 82efa4d5-f84d-4765-a042-3cbb8c544041 has session id: 6892d9b7-9ef5-4f95-bdab-c03ff61fc150","v":0,"name":"crucible","level":30{"msg":","time":"[1] Proc runs for 127.0.0.1:40751 in state New"2023-09-22T23:21:37.69773119Z",",v":"0hostname",":name":""crucible","level":ip-10-150-1-55.us-west-2.compute.internal"30,"pid":4769}
,"time":"2023-09-22T23:21:37.697758764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 8e88d9c1-84f3-4981-affe-0c1310d04226 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.697799666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2"}
The guest has requested activation
Sep 22 23:21:37.697 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.697 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.698 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.698 INFO Connection request from 1c48f237-34b8-4484-b38b-7c6b80300cc8 with version 4, task: proc
{"msg":"[0] Proc runs for 127.0.0.1:43936 in state New","v":0,"name":"crucible","level":30,"time":"Sep 22 23:21:37.698 INFO upstairs UpstairsConnection { upstairs_id: 1c48f237-34b8-4484-b38b-7c6b80300cc8, session_id: 126b8002-1d8d-46c3-951e-15087f60a7c9, gen: 1 } connected, version 4, task: proc
2023-09-22T23:21:37.698097675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.698158596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1"}
{"msg":"[1] Proc runs for 127.0.0.1:39320 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.698203866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d looper connected","v":0Sep 22 23:21:37.698 INFO accepted connection from 127.0.0.1:51763, task: main
,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.698262618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2"}
{{"msg":""msg"{:[2] Proc runs for 127.0.0.1:57826 in state New{Sep 22 23:21:37.698 INFO Connection request from f712941d-0577-43e4-8a2e-7814ec270c09 with version 4, task: proc
""""msg,"":"v"[0] connecting to 127.0.0.1:39150msg""Upstairs starts:"0,:"",,""v[2] Proc runs for 127.0.0.1:42674 in state New"":vname,0"""::0",,v""name:"0:",crucible""name",:"""cruciblename"":,""cruciblelevelSep 22 23:21:37.698 INFO upstairs UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } connected, version 4, task: proc
levelcrucible"":,"30"":,level30""level"::3030,"time":","2023-09-22T23:21:37.698351182Z"time,"":,time"":""2023-09-22T23:21:37.698346874Zhostname""2023-09-22T23:21:37.698353774Z,"":",hostname"":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid"ip-10-150-1-55.us-west-2.compute.internal"":,,4769""pidpid""::47694769}
,"time"Sep 22 23:21:37.698 INFO Connection request from f712941d-0577-43e4-8a2e-7814ec270c09 with version 4, task: proc
:"2023-09-22T23:21:37.698354057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid,":"4769looper":"0"}
}
{{"msg":""msg":"Sep 22 23:21:37.698 INFO upstairs UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } connected, version 4, task: proc
[1] connecting to 127.0.0.1:64794Crucible Version: BuildInfo {"\n,    version: {\""v0.0.1"\":,\n0,"    git_sha: \"name":""crucibleed48f294784d46ea7d4bb99336918b74358eca46"\"msg,,\n""level"    git_commit_timestamp: ::30\""2023-09-22T22:51:18.000000000Z\",\n[2] 1c48f237-34b8-4484-b38b-7c6b80300cc8 looper connected"    git_branch: \"main\",\n,"v"    rustc_semver: \":1.70.0\"0,\n,"name"    rustc_channel: \":"stable\"crucible",\n,"level"    rustc_host_triple: ,\"":timex86_64-unknown-illumos"\":,"\n302023-09-22T23:21:37.698501932Z    rustc_commit_sha: "\","hostname":90c541806f23a127002de5b4038be731ba1458ca"\",\n    cargo_triple: \"ip-10-150-1-55.us-west-2.compute.internal","x86_64-unknown-illumospid\"":,\n4769    debug: true,,\n"looper"    opt_level: 0,:\n"}1""}}
,"
{v":0,""namemsg""::"",crucible"",[2] connecting to 127.0.0.1:43168""time,level"""v:"30::0","name":"2023-09-22T23:21:37.698525651Z"crucible",",level":"30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","}time,"":"
2023-09-22T23:21:37.698551933Ztime"":","hostname"2023-09-22T23:21:37.698558293Z:"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769ip-10-150-1-55.us-west-2.compute.internal","}pid
":4769,"looper":Sep 22 23:21:37.698 INFO Connection request from f712941d-0577-43e4-8a2e-7814ec270c09 with version 4, task: proc
"{2""msg"}:"
Upstairs <-> Downstairs Message Version: 4","v"{:0,""name"msg:"":"crucible",up_listen starts""level,"":v"30:0,"name":"crucible","level":30,"time",:""Sep 22 23:21:37.698 INFO upstairs UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } connected, version 4, task: proc
2023-09-22T23:21:37.6986053Ztime"",:""hostname":"2023-09-22T23:21:37.698608964Z","ip-10-150-1-55.us-west-2.compute.internalhostname"":,""pid":4769}ip-10-150-1-55.us-west-2.compute.internal
","pid":4769{,"task":"up_listen""msg}"
:"{"Crucible stats registered with UUID: 7be22e7c-d55a-4790-a835-081c096012f4"msg":,""v":0,"nameWait for all three downstairs to come online"":","crucible"v",:"0,level"":name"30:"crucible","level":30,"time":","time":"2023-09-22T23:21:37.698655639Z","2023-09-22T23:21:37.69865806Z"hostname":,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"ip-10-150-1-55.us-west-2.compute.internal:"4769,"pid}"
:4769}
{{""msg"msg:"":"Flush timeout: 0.5","v":0,"name"Crucible 7be22e7c-d55a-4790-a835-081c096012f4 has session id: 354f8ce2-d4bb-4d47-a624-d406af786eb7":","crucible"v",:"0level,"":30name":"crucible","level":30,"time":","time":2023-09-22T23:21:37.698690815Z"","hostname2023-09-22T23:21:37.698693507Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4769,"pid}"
:4769}
{"msg":"82efa4d5-f84d-4765-a042-3cbb8c544041 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.698725045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] 82efa4d5-f84d-4765-a042-3cbb8c544041 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.698757626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
{"msg":"[0] Proc runs for 127.0.0.1:39150 in state New","v":0,"name{":"crucible","level":30"msg":"Upstairs starts","v":0,"name":"crucible",","time"level":":302023-09-22T23:21:37.698783312Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.698 INFO accepted connection from 127.0.0.1:52973, task: main
,"time":"2023-09-22T23:21:37.698798624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\nSep 22 23:21:37.698 INFO accepted connection from 127.0.0.1:43044, task: main
    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.698949491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":Sep 22 23:21:37.698 INFO [0] f712941d-0577-43e4-8a2e-7814ec270c09 (512f09eb-7e91-421e-9fa9-f1bb0acbe6ae) New New New ds_transition to WaitActive
"2023-09-22T23:21:37.698996661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible stats registered with UUID: dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.699044215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 has session id: 12ec540d-5ec3-4696-aaa4-c59d1d2545f8","v":0,"name":"crucible","level":30Sep 22 23:21:37.699 INFO [0] Transition from New to WaitActive
,"time":"2023-09-22T23:21:37.699083589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.699 INFO [0] client is_active_req TRUE, promote! session 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae
The guest has requested activation
The guest has requested activation
The guest has requested activation
Sep 22 23:21:37.699 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.699 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.699 INFO [1] f712941d-0577-43e4-8a2e-7814ec270c09 (512f09eb-7e91-421e-9fa9-f1bb0acbe6ae) WaitActive New New ds_transition to WaitActive
{Sep 22 23:21:37.699 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.699 INFO listening on 127.0.0.1:0, task: main
"msg":"[2] Proc runs for 127.0.0.1:59938 in state New","v":0,"name":"crucible","level":30Sep 22 23:21:37.699 INFO [1] client is_active_req TRUE, promote! session 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae
Sep 22 23:21:37.699 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.699 INFO Connection request from 1c48f237-34b8-4484-b38b-7c6b80300cc8 with version 4, task: proc
Sep 22 23:21:37.699 INFO Connection request from 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d with version 4, task: proc
,"time":"2023-09-22T23:21:37.699336772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.699 INFO upstairs UpstairsConnection { upstairs_id: 1c48f237-34b8-4484-b38b-7c6b80300cc8, session_id: 126b8002-1d8d-46c3-951e-15087f60a7c9, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.699 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.699 INFO upstairs UpstairsConnection { upstairs_id: 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d, session_id: 412fdf38-1b7f-416c-b371-8bd0b7f86f94, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.699 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.699 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.699 INFO Connection request from 8e88d9c1-84f3-4981-affe-0c1310d04226 with version 4, task: proc
Sep 22 23:21:37.699 INFO accepted connection from 127.0.0.1:44013, task: main
{"msg":"[1] 82efa4d5-f84d-4765-a042-3cbb8c544041 looper connected"Sep 22 23:21:37.699 INFO upstairs UpstairsConnection { upstairs_id: 8e88d9c1-84f3-4981-affe-0c1310d04226, session_id: d738ea25-1afc-4d4c-9b80-99c2ada34ff7, gen: 1 } connected, version 4, task: proc
,Sep 22 23:21:37.699 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.699 INFO [2] f712941d-0577-43e4-8a2e-7814ec270c09 (512f09eb-7e91-421e-9fa9-f1bb0acbe6ae) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.699 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.699 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.699 INFO Connection request from 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d with version 4, task: proc
Sep 22 23:21:37.699 INFO [2] client is_active_req TRUE, promote! session 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae
Sep 22 23:21:37.699 INFO Connection request from 8e88d9c1-84f3-4981-affe-0c1310d04226 with version 4, task: proc
{"msg":"[0] connecting to 127.0.0.1:63497","v":0,"name":"crucible","level":30Sep 22 23:21:37.699 INFO upstairs UpstairsConnection { upstairs_id: 8e88d9c1-84f3-4981-affe-0c1310d04226, session_id: d738ea25-1afc-4d4c-9b80-99c2ada34ff7, gen: 1 } connected, version 4, task: proc
,"time":"2023-09-22T23:21:37.699596388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
{"msg":"[1] connecting to 127.0.0.1:62341","v":0,"name":"crucible","level":30,"time"Sep 22 23:21:37.699 INFO Connection request from 1c48f237-34b8-4484-b38b-7c6b80300cc8 with version 4, task: proc
:"2023-09-22T23:21:37.699649102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1"}
Sep 22 23:21:37.699 INFO upstairs UpstairsConnection { upstairs_id: 1c48f237-34b8-4484-b38b-7c6b80300cc8, session_id: 126b8002-1d8d-46c3-951e-15087f60a7c9, gen: 1 } connected, version 4, task: proc
{{"msg":"[0] connecting to 127.0.0.1:54884","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.699804643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
Sep 22 23:21:37.699 INFO UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } is now active (read-write)
{""vmsg":"[0] 1c48f237-34b8-4484-b38b-7c6b80300cc8 (126b8002-1d8d-46c3-951e-15087f60a7c9) New New New ds_transition to WaitActive",Sep 22 23:21:37.700 INFO accepted connection from 127.0.0.1:47996, task: main
"v":0,"name":"crucible","level":30Sep 22 23:21:37.699 INFO Connection request from 8e88d9c1-84f3-4981-affe-0c1310d04226 with version 4, task: proc
":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.700108304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.700 INFO UpstairsConnection { upstairs_id: 1c48f237-34b8-4484-b38b-7c6b80300cc8, session_id: 126b8002-1d8d-46c3-951e-15087f60a7c9, gen: 1 } is now active (read-write)
Sep 22 23:21:37.700 INFO accepted connection from 127.0.0.1:59016, task: main
{Sep 22 23:21:37.700 INFO UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } is now active (read-write)
"msg"Sep 22 23:21:37.700 INFO accepted connection from 127.0.0.1:51535, task: main
Sep 22 23:21:37.700 INFO upstairs UpstairsConnection { upstairs_id: 8e88d9c1-84f3-4981-affe-0c1310d04226, session_id: d738ea25-1afc-4d4c-9b80-99c2ada34ff7, gen: 1 } connected, version 4, task: proc
:""[0] Transition from New to WaitActive","v":0msg"{,":name"":"crucible"[0] connecting to 127.0.0.1:34554"","levelmsg"",:":30"v":0,[2] connecting to 127.0.0.1:51713""name":",crucible"","v"level:":030,"name":"crucible","level":Sep 22 23:21:37.700 INFO accepted connection from 127.0.0.1:33771, task: main
30,"time":"2023-09-22T23:21:37.70019406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",,""time"pid":":47692023-09-22T23:21:37.700203658Z"},
","hostname":time"":"{2023-09-22T23:21:37.700212899Z"ip-10-150-1-55.us-west-2.compute.internal",",pid""":hostname"msg":4769":","looper"ip-10-150-1-55.us-west-2.compute.internal":",[0] client is_active_req TRUE, promote! session 126b8002-1d8d-46c3-951e-15087f60a7c9"0"",pid"":}v"
4769Sep 22 23:21:37.700 INFO accepted connection from 127.0.0.1:34244, task: main
:,"0Sep 22 23:21:37.700 INFO UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } is now active (read-write)
{,Sep 22 23:21:37.700 INFO accepted connection from 127.0.0.1:36546, task: main
"msg":"[1] connecting to 127.0.0.1:59903","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.700331435Z","hostname":"Sep 22 23:21:37.700 INFO accepted connection from 127.0.0.1:54123, task: main
ip-10-150-1-55.us-west-2.compute.internal","pid":"4769name",":looper"":"crucible"1",}"Sep 22 23:21:37.700 INFO UpstairsConnection { upstairs_id: 1c48f237-34b8-4484-b38b-7c6b80300cc8, session_id: 126b8002-1d8d-46c3-951e-15087f60a7c9, gen: 1 } is now active (read-write)
level"
:30{"Sep 22 23:21:37.700 INFO accepted connection from 127.0.0.1:54296, task: main
msg":"[2] connecting to 127.0.0.1:36502","v":0,"name":"crucible","level",:"30time":"2023-09-22T23:21:37.700383427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"{2023-09-22T23:21:37.700403744Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2"[1] 1c48f237-34b8-4484-b38b-7c6b80300cc8 (126b8002-1d8d-46c3-951e-15087f60a7c9) WaitActive New New ds_transition to WaitActive"}
,"v":0,"name":"{crucible","level":"30msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.700443433Z","hostname":","time":ip-10-150-1-55.us-west-2.compute.internal"","pid":2023-09-22T23:21:37.700450785Z"4769,"hostname"}:
"ip-10-150-1-55.us-west-2.compute.internal","pid":{4769,"task":""up_listen"msg"}:
"[1] Transition from New to WaitActive","v":{Sep 22 23:21:37.700 INFO accepted connection from 127.0.0.1:53816, task: main
0"msg",:""name":"crucible",Wait for all three downstairs to come online""level",:"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.700511179Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid2023-09-22T23:21:37.700516003Z"":,4769"hostname":}"
Sep 22 23:21:37.700 INFO UpstairsConnection { upstairs_id: 1c48f237-34b8-4484-b38b-7c6b80300cc8, session_id: 126b8002-1d8d-46c3-951e-15087f60a7c9, gen: 1 } is now active (read-write)
ip-10-150-1-55.us-west-2.compute.internal"{,""pid"msg"::4769"}
[1] client is_active_req TRUE, promote! session 126b8002-1d8d-46c3-951e-15087f60a7c9","v":0,"name"{:"crucible","level":"30msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.700566161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4769"time"}:"
2023-09-22T23:21:37.700573835Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4769"msg}"
:"{[2] 1c48f237-34b8-4484-b38b-7c6b80300cc8 (126b8002-1d8d-46c3-951e-15087f60a7c9) WaitActive WaitActive New ds_transition to WaitActive""msg",:""v":0,"name":"8f69534d-528b-4c23-b2c9-ce2dfe832ae1 active request set"crucible",,""v"level"::030,"name":"crucible","level":30Sep 22 23:21:37.700 INFO UpstairsConnection { upstairs_id: 8e88d9c1-84f3-4981-affe-0c1310d04226, session_id: d738ea25-1afc-4d4c-9b80-99c2ada34ff7, gen: 1 } is now active (read-write)
,",time":""time":"2023-09-22T23:21:37.700633873Z"2023-09-22T23:21:37.700637429Z",",hostname"":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid"":,"4769pid":4769}
}
{"{msg":"[2] Transition from New to WaitActive""msg":,""v":0,"name":"[0] 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 looper connected"crucible",,""v"level"::030,"name":"crucible","level":30looper":"2"}
,"time":","time2023-09-22T23:21:37.700695573Z"":",{"2023-09-22T23:21:37.700699128Z",{"hostname",msg""time":msghostname"""::":"""ip-10-150-1-55.us-west-2.compute.internalup_listen starts:"ip-10-150-1-55.us-west-2.compute.internal",","""2023-09-22T23:21:37.700127898Zpid""",pid":"4769,[0] 8e88d9c1-84f3-4981-affe-0c1310d04226 (d738ea25-1afc-4d4c-9b80-99c2ada34ff7) New New New ds_transition to WaitActive:"4769,""},vlooper
"":vhostname""::"{"0"0"ip-10-150-1-55.us-west-2.compute.internal,:}"0",""msg",Sep 22 23:21:37.700 INFO UpstairsConnection { upstairs_id: 8e88d9c1-84f3-4981-affe-0c1310d04226, session_id: d738ea25-1afc-4d4c-9b80-99c2ada34ff7, gen: 1 } is now active (read-write)
:
namename""pid{":""::"4769crucible"""crucible,",[2] client is_active_req TRUE, promote! session 126b8002-1d8d-46c3-951e-15087f60a7c9msg,""":looper","":v"level""level"1:"":30:"0[0] Proc runs for 127.0.0.1:34554 in state New",}30
","name"v":":crucible"0{,,""name"level"::""crucible"msg,,,""level":""time30"time"::""":2023-09-22T23:21:37.70081188Z2023-09-22T23:21:37.700816504Z""30,,""[1] Proc runs for 127.0.0.1:64794 in state Newhostnamehostname""::""","v"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"":,,""pid"pid:"4769:04769,,"",name"":tasktime"":crucible","""}:
"time"up_listen":,2023-09-22T23:21:37.700838998Z"""{}level2023-09-22T23:21:37.700844571Z"
",msg,{"""":msg""::30hostname"hostname"":Sep 22 23:21:37.700 INFO UpstairsConnection { upstairs_id: 8e88d9c1-84f3-4981-affe-0c1310d04226, session_id: d738ea25-1afc-4d4c-9b80-99c2ada34ff7, gen: 1 } is now active (read-write)
"ip-10-150-1-55.us-west-2.compute.internal":Wait for all three downstairs to come online"",",[0] Transition from New to WaitActive""v",:"0v,ip-10-150-1-55.us-west-2.compute.internal""",pid"time:","":name04769"pid"::"}",:""name"crucible:""
2023-09-22T23:21:37.700892003Z"crucible,"",level,""4769hostname":"30:level"}:"
ip-10-150-1-55.us-west-2.compute.internal"30,"pid":4769,"{time"}:
"",msg":"2023-09-22T23:21:37.700940437Z""time",:""hostname{"2023-09-22T23:21:37.700948898Z:""[1] 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 looper connected","ip-10-150-1-55.us-west-2.compute.internal"msghostname"",":""pid:,""ip-10-150-1-55.us-west-2.compute.internal"v:"4769[2] 82efa4d5-f84d-4765-a042-3cbb8c544041 looper connected"},
"",pid{"":v"":msg4769"0}::
"0,,{Flush timeout: 0.5"",""name"vmsg""::"0:,""crucible"[0] client is_active_req TRUE, promote! session d738ea25-1afc-4d4c-9b80-99c2ada34ff7name"",name,""::"""cruciblev""level""crucible":,0:,""levelname""30,::"30"crucible"level",:"30level":30,"time":"2023-09-22T23:21:37.701020877Z","hostname,"":"time":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:21:37.701025631Z"",","pid",:hostname4769":"}"
ip-10-150-1-55.us-west-2.compute.internal,time{""":time"",""msg"pid:2023-09-22T23:21:37.70101971Z"""::"4769,2023-09-22T23:21:37.70102499Z}""dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 active request set
"hostname,""{v,:""""msg:"ip-10-150-1-55.us-west-2.compute.internal":hostname",{"":pid"":0",4769msg"ip-10-150-1-55.us-west-2.compute.internal",",name[1] 8e88d9c1-84f3-4981-affe-0c1310d04226 (d738ea25-1afc-4d4c-9b80-99c2ada34ff7) WaitActive New New ds_transition to WaitActive"":"",crucible"""pid,v"looper::4769"":"0level,",:"30:name"":"""looper"2":[0] downstairs client at 127.0.0.1:46967 has UUID ff162497-e5a3-4763-b129-29dec4869786""}crucible
,"",time"1level,""""}{v::"
302023-09-22T23:21:37.701106078Z""",msg":":"hostname"{0:[2] Proc runs for 127.0.0.1:43168 in state New"",",""timeip-10-150-1-55.us-west-2.compute.internal",msg"":,v""pid":2023-09-22T23:21:37.701135053Z"""":name"0:,:",[1] Proc runs for 127.0.0.1:59903 in state New"hostnamename""4769,":"":"v"crucible"ip-10-150-1-55.us-west-2.compute.internalcrucible",,"":"}0level
,,"{"""pid"msg"level"::4769Sep 22 23:21:37.701 INFO Connection request from 7be22e7c-d55a-4790-a835-081c096012f4 with version 4, task: proc
name"":}"
"::30crucible"30[0] dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 looper connected{",",""msgv""::"0level,""[1] Transition from New to WaitActivename"",:"":vcrucible"":300,",level""Sep 22 23:21:37.701 INFO upstairs UpstairsConnection { upstairs_id: 7be22e7c-d55a-4790-a835-081c096012f4, session_id: 8fe9d3ab-892b-48e5-a656-2b4564d918d0, gen: 1 } connected, version 4, task: proc
:name30":",,""crucible"time"time,":"":,2023-09-22T23:21:37.701206725Z"level,",:"30time""":time"""hostname:""2023-09-22T23:21:37.701208023Z"2023-09-22T23:21:37.701224995Z"2023-09-22T23:21:37.701218764Z",,,,:""time""""hostnamehostname":"":ip-10-150-1-55.us-west-2.compute.internal":2023-09-22T23:21:37.701240522Z""",,ip-10-150-1-55.us-west-2.compute.internal""hostname"hostnamepid"":ip-10-150-1-55.us-west-2.compute.internal""4769,:,:}""""pid"
ip-10-150-1-55.us-west-2.compute.internal":ip-10-150-1-55.us-west-2.compute.internalpid"",:4769,4769""pidpid,"}"
:":4769looper4769":"}0
"{}}{
""
msg"msg":{"{:"Sep 22 23:21:37.701 INFO Connection request from 7be22e7c-d55a-4790-a835-081c096012f4 with version 4, task: proc
[2] 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 looper connected""[1] client is_active_req TRUE, promote! session d738ea25-1afc-4d4c-9b80-99c2ada34ff7"msg"msg"":,"",v:[0] Proc runs for 127.0.0.1:63497 in state New""":"{,Sep 22 23:21:37.699 INFO Connection request from 82efa4d5-f84d-4765-a042-3cbb8c544041 with version 4, task: proc
""v"msg:"0:,[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ff162497-e5a3-4763-b129-29dec4869786, encrypted: true, database_read_version: 1, database_write_version: 1 }"""name":"[1] connecting to 127.0.0.1:38174"crucible",,"",v"":v"0:level",:0"30name":","crucible"name",:""level":crucible"30,"level":30,Sep 22 23:21:37.701 INFO upstairs UpstairsConnection { upstairs_id: 82efa4d5-f84d-4765-a042-3cbb8c544041, session_id: 13fb2768-edc0-4158-a955-c73279bce790, gen: 1 } connected, version 4, task: proc
"time":"2023-09-22T23:21:37.701382591Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid":2023-09-22T23:21:37.701389311Z"4769Sep 22 23:21:37.701 INFO [0] downstairs client at 127.0.0.1:46295 has UUID ca34832d-be06-44a7-9364-f6c3bc44d7fb
,"},
hostname""time{0:"""msg,:"ip-10-150-1-55.us-west-2.compute.internal""":"name,2023-09-22T23:21:37.701392161Z"""[1] dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 looper connected,"pid","":v""Sep 22 23:21:37.701 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ca34832d-be06-44a7-9364-f6c3bc44d7fb, encrypted: true, database_read_version: 1, database_write_version: 1 }
:4769:crucible"hostname","0level",:,:"30""name":"looper"crucible":,ip-10-150-1-55.us-west-2.compute.internal""1"",level"":pid30":}4769
}
,{"time":""2023-09-22T23:21:37.701453921Z"{msg":,",""hostname""msg:[2] connecting to 127.0.0.1:34655time":"":"""ip-10-150-1-55.us-west-2.compute.internal,"2023-09-22T23:21:37.701465019Z"1c48f237-34b8-4484-b38b-7c6b80300cc8 WaitActive WaitActive WaitActivev,""Sep 22 23:21:37.701 INFO f712941d-0577-43e4-8a2e-7814ec270c09 WaitActive WaitActive WaitActive
":Sep 22 23:21:37.701 INFO Connection request from 82efa4d5-f84d-4765-a042-3cbb8c544041 with version 4, task: proc
0,v":0,"name":"crucible","level":30Sep 22 23:21:37.701 INFO upstairs UpstairsConnection { upstairs_id: 82efa4d5-f84d-4765-a042-3cbb8c544041, session_id: 13fb2768-edc0-4158-a955-c73279bce790, gen: 1 } connected, version 4, task: proc
"name":"crucible","level":30Sep 22 23:21:37.701 INFO upstairs UpstairsConnection { upstairs_id: 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d, session_id: 412fdf38-1b7f-416c-b371-8bd0b7f86f94, gen: 1 } connected, version 4, task: proc
,"time":"2023-09-22T23:21:37.701536164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,,""looper":"v2"",}:"
time"Sep 22 23:21:37.701 INFO Connection request from dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 with version 4, task: proc
0,:"{"Sep 22 23:21:37.701 INFO [1] downstairs client at 127.0.0.1:43385 has UUID f7971d88-fcac-4a34-864e-16881c021a65
2023-09-22T23:21:37.701550767Z"name"",:"msg"hostname:"":""crucible"[2] Proc runs for 127.0.0.1:36502 in state New","ip-10-150-1-55.us-west-2.compute.internal",level",v"":Sep 22 23:21:37.701 INFO upstairs UpstairsConnection { upstairs_id: dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9, session_id: 9165874c-5197-4919-861e-2430574d8700, gen: 1 } connected, version 4, task: proc
"pid"::Sep 22 23:21:37.701 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f7971d88-fcac-4a34-864e-16881c021a65, encrypted: true, database_read_version: 1, database_write_version: 1 }
4769,30"0looper",:""name"2":"}crucible"
,"level":30Sep 22 23:21:37.701 INFO Connection request from 82efa4d5-f84d-4765-a042-3cbb8c544041 with version 4, task: proc
,{"Sep 22 23:21:37.701 INFO Connection request from 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d with version 4, task: proc
time""msg"::""up_listen starts"2023-09-22T23:21:37.701617566Z",,","v""hostname"time":::""0,"2023-09-22T23:21:37.701631993Z"ip-10-150-1-55.us-west-2.compute.internal"Sep 22 23:21:37.701 INFO upstairs UpstairsConnection { upstairs_id: 82efa4d5-f84d-4765-a042-3cbb8c544041, session_id: 13fb2768-edc0-4158-a955-c73279bce790, gen: 1 } connected, version 4, task: proc
,nameSep 22 23:21:37.701 INFO f712941d-0577-43e4-8a2e-7814ec270c09 WaitActive WaitActive WaitActive
":Sep 22 23:21:37.701 INFO upstairs UpstairsConnection { upstairs_id: 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d, session_id: 412fdf38-1b7f-416c-b371-8bd0b7f86f94, gen: 1 } connected, version 4, task: proc
",pid"""crucible"hostname"::","4769level":ip-10-150-1-55.us-west-2.compute.internal"30},
"pid":4769}
{,""time"msg"::""2023-09-22T23:21:37.701686899Z","hostname":[1] downstairs client at 127.0.0.1:55221 has UUID d803d148-f26b-4671-a1b7-af69788d7c3f"","v":Sep 22 23:21:37.701 INFO Connection request from dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 with version 4, task: proc
0,ip-10-150-1-55.us-west-2.compute.internal"","pid":4769,"task":"name"up_listen":}"
crucible","pidlevel"":{:304769"Sep 22 23:21:37.701 INFO upstairs UpstairsConnection { upstairs_id: dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9, session_id: 9165874c-5197-4919-861e-2430574d8700, gen: 1 } connected, version 4, task: proc
msg":"Sep 22 23:21:37.701 INFO [2] downstairs client at 127.0.0.1:64157 has UUID 4afe10fd-86f8-4da0-8ac1-1f760ded452a
Wait for all three downstairs to come online}",,""timev
""::"0,"{2023-09-22T23:21:37.701737215Z"name":","crucible"hostname",":""levelmsg"":ip-10-150-1-55.us-west-2.compute.internal":"30Sep 22 23:21:37.701 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4afe10fd-86f8-4da0-8ac1-1f760ded452a, encrypted: true, database_read_version: 1, database_write_version: 1 }
,"pid":4769[2] 8e88d9c1-84f3-4981-affe-0c1310d04226 (d738ea25-1afc-4d4c-9b80-99c2ada34ff7) WaitActive WaitActive New ds_transition to WaitActive"}
,"v":0,"name":"{crucible",,""time"level":":"30msg"2023-09-22T23:21:37.701774668Z":","Sep 22 23:21:37.701 INFO Current flush_numbers [0..12]: [0, 0]
[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d803d148-f26b-4671-a1b7-af69788d7c3f, encrypted: true, database_read_version: 1, database_write_version: 1 }"hostname,"":v",:"0"time",":ip-10-150-1-55.us-west-2.compute.internal"name"":,{"""{msg":"[0] 82efa4d5-f84d-4765-a042-3cbb8c544041 (13fb2768-edc0-4158-a955-c73279bce790) New New New ds_transition to WaitActive","v":0,""name":"cruciblecrucible"","level",:"30level":30"msg":"[0] 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d (412fdf38-1b7f-416c-b371-8bd0b7f86f94) New New New ds_transition to WaitActive","v":0,"name":"crucible",",level"",:time"":"time30"2023-09-22T23:21:37.701907289Z":","hostname"2023-09-22T23:21:37.701908717Z":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal4769","pid"}:
4769,"time":"}
Sep 22 23:21:37.701 INFO Current flush_numbers [0..12]: [0, 0]
2023-09-22T23:21:37.701933626Z"{{,"hostname":"""ip-10-150-1-55.us-west-2.compute.internal"msg"msg,":pid"""::"4769[0] Transition from New to WaitActive","}v1c48f237-34b8-4484-b38b-7c6b80300cc8 WaitActive WaitActive WaitActive
"":,0",v"{"name"::"0crucible"",,""msgname""level":"::"30crucible","[0] Transition from New to WaitActive"level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.701999981Z","hostname":,""time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:21:37.702003899Z",pid"","time"::4769"hostname":"}2023-09-22T23:21:37.70200907Z"
,ip-10-150-1-55.us-west-2.compute.internal"","hostname"pid"::{4769"}
ip-10-150-1-55.us-west-2.compute.internal"","msg"pid"::"4769}
[0] client is_active_req TRUE, promote! session 13fb2768-edc0-4158-a955-c73279bce790","v":0,"{name":"crucible","level"":msg"30:"[0] client is_active_req TRUE, promote! session 412fdf38-1b7f-416c-b371-8bd0b7f86f94","v":0,"name":"crucible","level":30,"time":"Sep 22 23:21:37.702 INFO Downstairs has completed Negotiation, task: proc
2023-09-22T23:21:37.702067403Z","hostname":"Sep 22 23:21:37.701 INFO upstairs UpstairsConnection { upstairs_id: 7be22e7c-d55a-4790-a835-081c096012f4, session_id: 8fe9d3ab-892b-48e5-a656-2b4564d918d0, gen: 1 } connected, version 4, task: proc
ip-10-150-1-55.us-west-2.compute.internal",,""time"pid"::"4769}2023-09-22T23:21:37.702077668Z
","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":"4769msg":"}
Sep 22 23:21:37.702 INFO Connection request from 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 with version 4, task: proc
{[1] 82efa4d5-f84d-4765-a042-3cbb8c544041 (13fb2768-edc0-4158-a955-c73279bce790) WaitActive New New ds_transition to WaitActive",""v"msg:"0:,"Sep 22 23:21:37.702 INFO UpstairsConnection { upstairs_id: 82efa4d5-f84d-4765-a042-3cbb8c544041, session_id: 13fb2768-edc0-4158-a955-c73279bce790, gen: 1 } is now active (read-write)
"[1] 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d (412fdf38-1b7f-416c-b371-8bd0b7f86f94) WaitActive New New ds_transition to WaitActive"name":","crucible"v":,"0level",:"30name":Sep 22 23:21:37.702 INFO UpstairsConnection { upstairs_id: 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d, session_id: 412fdf38-1b7f-416c-b371-8bd0b7f86f94, gen: 1 } is now active (read-write)
"crucible"Sep 22 23:21:37.702 INFO upstairs UpstairsConnection { upstairs_id: 8f69534d-528b-4c23-b2c9-ce2dfe832ae1, session_id: 4c909eb0-0138-456e-a20b-76c3a0a91bc8, gen: 1 } connected, version 4, task: proc
,Sep 22 23:21:37.702 INFO Downstairs has completed Negotiation, task: proc
",level"":time"30:"2023-09-22T23:21:37.702146908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"{time":""2023-09-22T23:21:37.702171816Z"msg":","hostname":"[1] Transition from New to WaitActive","v"ip-10-150-1-55.us-west-2.compute.internal":,"0pid,""name"::4769"crucible"},
"Sep 22 23:21:37.702 INFO Connection request from 7be22e7c-d55a-4790-a835-081c096012f4 with version 4, task: proc
level{":"30msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,",time"":"hostname"2023-09-22T23:21:37.702216752Z":","hostname":"ip-10-150-1-55.us-west-2.compute.internal",",ip-10-150-1-55.us-west-2.compute.internaltime""":,""pidpid""Sep 22 23:21:37.702 INFO upstairs UpstairsConnection { upstairs_id: 7be22e7c-d55a-4790-a835-081c096012f4, session_id: 8fe9d3ab-892b-48e5-a656-2b4564d918d0, gen: 1 } connected, version 4, task: proc
2023-09-22T23:21:37.702225169Z"::4769,}Sep 22 23:21:37.702 INFO Connection request from 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 with version 4, task: proc

"Sep 22 23:21:37.702 INFO UpstairsConnection { upstairs_id: 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d, session_id: 412fdf38-1b7f-416c-b371-8bd0b7f86f94, gen: 1 } is now active (read-write)
hostname"{:""msg":"[1] client is_active_req TRUE, promote! session 13fb2768-edc0-4158-a955-c73279bce790","v":ip-10-150-1-55.us-west-2.compute.internal"0,,""name"pid"::"4769crucible"Sep 22 23:21:37.702 INFO upstairs UpstairsConnection { upstairs_id: 8f69534d-528b-4c23-b2c9-ce2dfe832ae1, session_id: 4c909eb0-0138-456e-a20b-76c3a0a91bc8, gen: 1 } connected, version 4, task: proc
},
"level":30{"msg":"[1] client is_active_req TRUE, promote! session 412fdf38-1b7f-416c-b371-8bd0b7f86f94","v":0,"name":"crucible","level":30,"time":"Sep 22 23:21:37.702 INFO UpstairsConnection { upstairs_id: 82efa4d5-f84d-4765-a042-3cbb8c544041, session_id: 13fb2768-edc0-4158-a955-c73279bce790, gen: 1 } is now active (read-write)
2023-09-22T23:21:37.702304478Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal,"","timepid""::"47692023-09-22T23:21:37.702319444Z"}
,"hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4769"}msg"
:"{"msg":"[2] 82efa4d5-f84d-4765-a042-3cbb8c544041 (13fb2768-edc0-4158-a955-c73279bce790) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible"[2] 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d (412fdf38-1b7f-416c-b371-8bd0b7f86f94) WaitActive WaitActive New ds_transition to WaitActive",",level"":v"30:0,"name":"crucible","level":30Sep 22 23:21:37.702 INFO Current flush_numbers [0..12]: [0, 0]
,"Sep 22 23:21:37.702 INFO Connection request from 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 with version 4, task: proc
,"{time"time"::"""2023-09-22T23:21:37.702379623Z"2023-09-22T23:21:37.702375019Z"msg",,:"""hostname"hostname"::""[2] downstairs client at 127.0.0.1:59938 has UUID 74dd6474-4427-416d-8d80-1f3912278411"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",,"",pid":"4769v"pid"}:
4769:Sep 22 23:21:37.702 INFO upstairs UpstairsConnection { upstairs_id: 8f69534d-528b-4c23-b2c9-ce2dfe832ae1, session_id: 4c909eb0-0138-456e-a20b-76c3a0a91bc8, gen: 1 } connected, version 4, task: proc
0,}"{name":
""cruciblemsg"":"{,"level":[2] Transition from New to WaitActive"30",Sep 22 23:21:37.702 INFO UpstairsConnection { upstairs_id: 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d, session_id: 412fdf38-1b7f-416c-b371-8bd0b7f86f94, gen: 1 } is now active (read-write)
Sep 22 23:21:37.702 INFO UpstairsConnection { upstairs_id: 82efa4d5-f84d-4765-a042-3cbb8c544041, session_id: 13fb2768-edc0-4158-a955-c73279bce790, gen: 1 } is now active (read-write)
"msg":v"":0,"[2] Transition from New to WaitActive"name":","crucible"v":,,""level"0:time"30:,""name":"crucible"2023-09-22T23:21:37.702441849Z","level",":hostname"30:"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.702465849Z",{"hostname":",""timemsg""ip-10-150-1-55.us-west-2.compute.internal":,"":"pid2023-09-22T23:21:37.702472599Z"":4769,"hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 74dd6474-4427-416d-8d80-1f3912278411, encrypted: true, database_read_version: 1, database_write_version: 1 }"pid":4769{,"v"}:
0","msg"name"::"{"crucible","level":"[2] client is_active_req TRUE, promote! session 13fb2768-edc0-4158-a955-c73279bce790msg"30",":v"":0,"name":"crucible"[2] client is_active_req TRUE, promote! session 412fdf38-1b7f-416c-b371-8bd0b7f86f94",","level"v"::300,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.702524536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"time"}:"
,Sep 22 23:21:37.702 INFO Downstairs has completed Negotiation, task: proc
"{time"msg":"":"1c48f237-34b8-4484-b38b-7c6b80300cc8 WaitActive WaitActive WaitActive","2023-09-22T23:21:37.702539716Zv"":,"hostname":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:21:37.702535045Z0,"",pid",:"4769"hostname"name"::""}crucible"
Sep 22 23:21:37.702 INFO Current flush_numbers [0..12]: [0, 0]
ip-10-150-1-55.us-west-2.compute.internal",",level"":pid"30:4769}
,"time":"2023-09-22T23:21:37.702611438Z","hostname":"2023-09-22T23:21:37.701795204Z"ip-10-150-1-55.us-west-2.compute.internal",",pid"":hostname"4769:"}
ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4769}
"msg":"{"msg"[0] 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 (4c909eb0-0138-456e-a20b-76c3a0a91bc8) New New New ds_transition to WaitActive:"","v"[2] Transition from New to WaitActive":,"0v",":name"0:","crucible"name":","crucible"level":,"30level":30,"time":","2023-09-22T23:21:37.702676545Z"time":","hostname":2023-09-22T23:21:37.70267779Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4769,"pid":}4769
}
{"msg"{:"[0] Transition from New to WaitActive"",msg"":v"":0,"name":"crucible"[2] client is_active_req TRUE, promote! session d738ea25-1afc-4d4c-9b80-99c2ada34ff7",","level"v"::300,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.702734136Z","hostname":,""time":"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:21:37.702738319Z"pid":4769,"hostname":}"
ip-10-150-1-55.us-west-2.compute.internal","pid":4769}{
"msg":"{[0] client is_active_req TRUE, promote! session 4c909eb0-0138-456e-a20b-76c3a0a91bc8"",msg":""v":0,"name":"crucible"[0] downstairs client at 127.0.0.1:36647 has UUID b778f7e5-519c-44d5-90ea-a92dbeffe813",",level"":v":Sep 22 23:21:37.702 INFO Downstairs has completed Negotiation, task: proc
30Sep 22 23:21:37.702 INFO Current flush_numbers [0..12]: [0, 0]
0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.702794285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"{time":""2023-09-22T23:21:37.702806229Z"msg":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769[1] 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 (4c909eb0-0138-456e-a20b-76c3a0a91bc8) WaitActive New New ds_transition to WaitActive",}"
v":0,"name":"crucible"{,"level":30"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b778f7e5-519c-44d5-90ea-a92dbeffe813, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,",name"":"time":crucible"","level":2023-09-22T23:21:37.702842373Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
pid{"msg":4769}
{{"msg":""msgFlush timeout: 0.5"":","v":0,"Sep 22 23:21:37.702 INFO Downstairs has completed Negotiation, task: proc
name"[0] downstairs client at 127.0.0.1:43936 has UUID 2ad3fd00-b9f7-450b-935d-a7c44c290e9e":"crucible",","vlevel""::30"0:","name":"crucible"[1] Transition from New to WaitActive","level",":v":300,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.702995113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,,""time":time{"":"2023-09-22T23:21:37.703007472Z""2023-09-22T23:21:37.703002874Z","msg"hostname,":"":hostname"":"7be22e7c-d55a-4790-a835-081c096012f4 active request set","ip-10-150-1-55.us-west-2.compute.internalv"":ip-10-150-1-55.us-west-2.compute.internal0",,""name,":""pid"pid"crucible:"4769:,"4769level"}:}30

{Sep 22 23:21:37.703 INFO UpstairsConnection { upstairs_id: 7be22e7c-d55a-4790-a835-081c096012f4, session_id: 8fe9d3ab-892b-48e5-a656-2b4564d918d0, gen: 1 } is now active (read-write)
"msg",:"{"time":"2023-09-22T23:21:37.703058195Z""msg":","hostname":"ip-10-150-1-55.us-west-2.compute.internal"[1] client is_active_req TRUE, promote! session 4c909eb0-0138-456e-a20b-76c3a0a91bc8[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2ad3fd00-b9f7-450b-935d-a7c44c290e9e, encrypted: true, database_read_version: 1, database_write_version: 1 }","",pid"",v":"4769:v"0},":
0name",:""name"crucible":{","crucible"level":,""30level"msg:":30"[0] 7be22e7c-d55a-4790-a835-081c096012f4 looper connected","v":0,"name":"crucible","level":30,",time"":"time":"2023-09-22T23:21:37.703117282Z"2023-09-22T23:21:37.703114711Z",Sep 22 23:21:37.703 INFO Current flush_numbers [0..12]: [0, 0]
"hostname",,:"""time"hostname"::ip-10-150-1-55.us-west-2.compute.internal""","pid"2023-09-22T23:21:37.703125675Zip-10-150-1-55.us-west-2.compute.internal:""4769,,""pid"hostname"}:
:"4769Sep 22 23:21:37.701 INFO f712941d-0577-43e4-8a2e-7814ec270c09 WaitActive WaitActive WaitActive
ip-10-150-1-55.us-west-2.compute.internal",}{
""pid":msg"4769:",{"looper":"0"}
[2] 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 (4c909eb0-0138-456e-a20b-76c3a0a91bc8) WaitActive WaitActive New ds_transition to WaitActive""msg,"":v":{"0,"name":""crucible"msg"69c4c5e1-2fd5-4777-bf6a-fcd75618f55d WaitActive WaitActive WaitActive",:","level"":v[0] Proc runs for 127.0.0.1:54884 in state New""30,":v"0:,"0name",:""Sep 22 23:21:37.703 INFO UpstairsConnection { upstairs_id: 7be22e7c-d55a-4790-a835-081c096012f4, session_id: 8fe9d3ab-892b-48e5-a656-2b4564d918d0, gen: 1 } is now active (read-write)
crucible"name":","crucible"level",:"30level":30,"time":"2023-09-22T23:21:37.703208937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,",{time":""time""2023-09-22T23:21:37.703225997Z:""msg",":"2023-09-22T23:21:37.703224696Z"hostname":",[2] Transition from New to WaitActive""ip-10-150-1-55.us-west-2.compute.internal"hostname,,"""v"::pid"":04769,"ip-10-150-1-55.us-west-2.compute.internal"name,}"
:""pid"crucible":,4769"Sep 22 23:21:37.703 INFO Current flush_numbers [0..12]: [0, 0]
{level"Sep 22 23:21:37.703 INFO Current flush_numbers [0..12]: [0, 0]
}:
"30msg":"{[1] 7be22e7c-d55a-4790-a835-081c096012f4 looper connected","v":0","msg"name"::""crucible","level":30[1] downstairs client at 127.0.0.1:39320 has UUID ab7c04fa-de73-4af6-8f95-d0dc6939a823","v":,0","timename""::""crucible"2023-09-22T23:21:37.703295275Z,"Sep 22 23:21:37.703 INFO Downstairs has completed Negotiation, task: proc
,""hostname",":"level"time"::Sep 22 23:21:37.703 INFO UpstairsConnection { upstairs_id: 7be22e7c-d55a-4790-a835-081c096012f4, session_id: 8fe9d3ab-892b-48e5-a656-2b4564d918d0, gen: 1 } is now active (read-write)
30"ip-10-150-1-55.us-west-2.compute.internal","2023-09-22T23:21:37.70330854Z"pid":,4769"hostname":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"{looper":",1"""}msg"
:time"":"Sep 22 23:21:37.703 INFO Current flush_numbers [0..12]: [0, 0]
{[2] client is_active_req TRUE, promote! session 4c909eb0-0138-456e-a20b-76c3a0a91bc8"2023-09-22T23:21:37.703333876Z"",msg"":,v""":hostname"0:[1] Proc runs for 127.0.0.1:38174 in state New","","name"vip-10-150-1-55.us-west-2.compute.internal:""",crucible":",0"pidlevel","::30"4769name":"crucible}"
,"level":30{"msg":,""time":"2023-09-22T23:21:37.703389033Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ab7c04fa-de73-4af6-8f95-d0dc6939a823, encrypted: true, database_read_version: 1, database_write_version: 1 }",}","
time"v:"":02023-09-22T23:21:37.703398721Z","name":,""hostname":crucible"","level":ip-10-150-1-55.us-west-2.compute.internal"30,"pid":4769}
{"msg":"[2] 7be22e7c-d55a-4790-a835-081c096012f4 looper connected","v":0,",name"":"time"crucible":,""level":2023-09-22T23:21:37.70343473Z"30,"hostname":"Sep 22 23:21:37.703 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.703 INFO Downstairs has completed Negotiation, task: proc
ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.703460464Z"{,"hostname":""msg"ip-10-150-1-55.us-west-2.compute.internal":","pid":4769,"69c4c5e1-2fd5-4777-bf6a-fcd75618f55d WaitActive WaitActive WaitActive"looper":",2""v"}:
0,"name":"crucible"{,"level":30"msg":"[2] Proc runs for 127.0.0.1:34655 in state New","v":0,"name":"crucible","level":30{,"time":"2023-09-22T23:21:37.703505405Z"","msghostname""::""ip-10-150-1-55.us-west-2.compute.internal",","timepid""::Sep 22 23:21:37.702 INFO UpstairsConnection { upstairs_id: 8f69534d-528b-4c23-b2c9-ce2dfe832ae1, session_id: 4c909eb0-0138-456e-a20b-76c3a0a91bc8, gen: 1 } is now active (read-write)
"[0] 1c48f237-34b8-4484-b38b-7c6b80300cc8 (126b8002-1d8d-46c3-951e-15087f60a7c9) WaitActive WaitActive WaitActive ds_transition to WaitQuorum"2023-09-22T23:21:37.703514929ZSep 22 23:21:37.703 INFO Downstairs has completed Negotiation, task: proc
",,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] 7be22e7c-d55a-4790-a835-081c096012f4 (8fe9d3ab-892b-48e5-a656-2b4564d918d0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.703580717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","Sep 22 23:21:37.701 INFO Connection request from dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 with version 4, task: proc
pid,"time":"2023-09-22T23:21:37.703591587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30":4769Sep 22 23:21:37.703 INFO upstairs UpstairsConnection { upstairs_id: dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9, session_id: 9165874c-5197-4919-861e-2430574d8700, gen: 1 } connected, version 4, task: proc
}
,{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30Sep 22 23:21:37.703 INFO UpstairsConnection { upstairs_id: 8f69534d-528b-4c23-b2c9-ce2dfe832ae1, session_id: 4c909eb0-0138-456e-a20b-76c3a0a91bc8, gen: 1 } is now active (read-write)
,"time"":"time":"2023-09-22T23:21:37.70366437Z",2023-09-22T23:21:37.703638612Z""hostname":","hostname":Sep 22 23:21:37.703 INFO Current flush_numbers [0..12]: [0, 0]
ip-10-150-1-55.us-west-2.compute.internal"Sep 22 23:21:37.703 INFO Current flush_numbers [0..12]: [0, 0]
","ip-10-150-1-55.us-west-2.compute.internalpid":4769}
{"msg":"[0] client is_active_req TRUE, promote! session 8fe9d3ab-892b-48e5-a656-2b4564d918d0","v":0,"name":"crucible","level":30","pid":4769}
,{""msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"time":"2023-09-22T23:21:37.703754703Z",2023-09-22T23:21:37.703725413Z""hostname":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pidip-10-150-1-55.us-west-2.compute.internal"":,"4769pid":4769}
}
Sep 22 23:21:37.703 INFO UpstairsConnection { upstairs_id: 8f69534d-528b-4c23-b2c9-ce2dfe832ae1, session_id: 4c909eb0-0138-456e-a20b-76c3a0a91bc8, gen: 1 } is now active (read-write)
{"msg":"[1] 7be22e7c-d55a-4790-a835-081c096012f4 (8fe9d3ab-892b-48e5-a656-2b4564d918d0) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,Sep 22 23:21:37.703 INFO Current flush_numbers [0..12]: [0, 0]
"time":"2023-09-22T23:21:37.703815115Z","hostname":"4769ip-10-150-1-55.us-west-2.compute.internal","pid":4769}}

{{""msgmsg"Sep 22 23:21:37.703 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.703 INFO Downstairs has completed Negotiation, task: proc
"::""[1] Transition from New to WaitActive","v":[2] downstairs client at 127.0.0.1:57826 has UUID 440ce80c-c0c9-48b9-ba41-98710683ab87"0,"name",:""v"crucible":,"0level",:"30name":"crucible","level":30,"time":"2023-09-22T23:21:37.703883983Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time,""pid":":47692023-09-22T23:21:37.703887713Z"}
,"hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":4769"msg":}"
[1] client is_active_req TRUE, promote! session 8fe9d3ab-892b-48e5-a656-2b4564d918d0","v":0,"name"{:"crucible","level":"30msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 440ce80c-c0c9-48b9-ba41-98710683ab87, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level",":time":"302023-09-22T23:21:37.703936771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{,""msg":"time":"2023-09-22T23:21:37.70395514Z","hostname":"[2] 7be22e7c-d55a-4790-a835-081c096012f4 (8fe9d3ab-892b-48e5-a656-2b4564d918d0) WaitActive WaitActive New ds_transition to WaitActive"ip-10-150-1-55.us-west-2.compute.internal",",v"":pid"0:,4769"name":"}
crucible","level":30{"msg":"69c4c5e1-2fd5-4777-bf6a-fcd75618f55d WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.70399868Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.704 INFO Downstairs has completed Negotiation, task: proc
,{"time":""msg":2023-09-22T23:21:37.704012705Z"","[2] Transition from New to WaitActive"hostname":","v":0ip-10-150-1-55.us-west-2.compute.internal",","namepid":"4769:"crucible"},"
level":30,"time":"2023-09-22T23:21:37.704052766Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","Sep 22 23:21:37.704 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.704 INFO Current flush_numbers [0..12]: [0, 0]
pid":4769}
{"msg":"[2] client is_active_req TRUE, promote! session 8fe9d3ab-892b-48e5-a656-2b4564d918d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.704104203Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":Sep 22 23:21:37.704 INFO UpstairsConnection { upstairs_id: dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9, session_id: 9165874c-5197-4919-861e-2430574d8700, gen: 1 } is now active (read-write)
4769}
,"time":"2023-09-22T23:21:37.702862262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"8e88d9c1-84f3-4981-affe-0c1310d04226 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.704188393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] downstairs client at 127.0.0.1:40751 has UUID 4e7888c2-7beb-4f8a-a2f2-dd1592f0ed78","v":0,"name":"crucible","level":30Sep 22 23:21:37.704 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.704 INFO Downstairs has completed Negotiation, task: proc
,Sep 22 23:21:37.704 INFO UpstairsConnection { upstairs_id: dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9, session_id: 9165874c-5197-4919-861e-2430574d8700, gen: 1 } is now active (read-write)
Sep 22 23:21:37.704 INFO Current flush_numbers [0..12]: [0, 0]
"time":"2023-09-22T23:21:37.704233977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4e7888c2-7beb-4f8a-a2f2-dd1592f0ed78, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.70430115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"8e88d9c1-84f3-4981-affe-0c1310d04226 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.704346062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] downstairs client at 127.0.0.1:42674 has UUID 7bf7d359-28ba-4346-a3ee-541f5bc5a1c0","v":0,"name":"crucible","level":304769,"looper":"1"}
,"time":"2023-09-22T23:21:37.704389982Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid":"4769msg":"}
[1] Proc runs for 127.0.0.1:62341 in state New","v":0{,"name":"crucible"",msg":""level":30[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7bf7d359-28ba-4346-a3ee-541f5bc5a1c0, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30Sep 22 23:21:37.704 INFO Downstairs has completed Negotiation, task: proc
,"time":",2023-09-22T23:21:37.704434541Z""time":","hostname":"2023-09-22T23:21:37.70444842Z","hostname"ip-10-150-1-55.us-west-2.compute.internal":","pid":4769ip-10-150-1-55.us-west-2.compute.internal","}pid
":4769}
{"{msg":""msg"[2] dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 looper connected":","v":0,"name":8e88d9c1-84f3-4981-affe-0c1310d04226 WaitActive WaitActive WaitActive"",crucible""v",:"0level,""name"::"30crucible","level":30,","time":time"":"2023-09-22T23:21:37.704505431Z"2023-09-22T23:21:37.704502787Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",",pid"":pid"4769:4769}
,"looper":"2"}
{{"{msg"":"msg"":"msg":"[2] Proc runs for 127.0.0.1:51713 in state New",[0] 8e88d9c1-84f3-4981-affe-0c1310d04226 (d738ea25-1afc-4d4c-9b80-99c2ada34ff7) WaitActive WaitActive WaitActive ds_transition to WaitQuorum""v","[0] 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d (412fdf38-1b7f-416c-b371-8bd0b7f86f94) WaitActive WaitActive WaitActive ds_transition to WaitQuorumv:""0:,",0name,""v""name":"::crucible"0",,crucible""",level"":name"level"::"3030crucible","level":30,"time":"2023-09-22T23:21:37.704592437Z","hostname":",,""ip-10-150-1-55.us-west-2.compute.internal"timetime"",::""pid"":2023-09-22T23:21:37.704595872Z47692023-09-22T23:21:37.704593086Z""},
,""hostname"hostname":":"{ip-10-150-1-55.us-west-2.compute.internal",ip-10-150-1-55.us-west-2.compute.internal""",msg""pid"pid:":":47694769[0] Transition from WaitActive to WaitQuorum"}}

,Sep 22 23:21:37.704 INFO [0] f712941d-0577-43e4-8a2e-7814ec270c09 (512f09eb-7e91-421e-9fa9-f1bb0acbe6ae) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
"v{"{:0,""name":"msg"crucible:"",""msg"level"::"[0] Transition from WaitActive to WaitQuorum"Sep 22 23:21:37.704 INFO [0] Transition from WaitActive to WaitQuorum
30Sep 22 23:21:37.704 INFO Current flush_numbers [0..12]: [0, 0]
[0] dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 (9165874c-5197-4919-861e-2430574d8700) New New New ds_transition to WaitActive",,""v"v:",0":,"timename":0"",crucible":""nameSep 22 23:21:37.704 WARN [0] new RM replaced this: None
2023-09-22T23:21:37.704680405Z","":level","":crucible"30hostname",:""level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{,""msg":time""Sep 22 23:21:37.704 INFO Current flush_numbers [0..12]: [0, 0]
:",[0] new RM replaced this: None""2023-09-22T23:21:37.704714149Z",time,"":""v"hostname2023-09-22T23:21:37.704718329Z""::0",,""nameip-10-150-1-55.us-west-2.compute.internalhostname":""",":pid"ip-10-150-1-55.us-west-2.compute.internal":",4769"crucible"pid"},:
4769"level":}40
{"msg"{:"[0] new RM replaced this: None",""msgv""::"Sep 22 23:21:37.704 INFO [0] Starts reconcile loop
,"[0] Transition from New to WaitActive"0time,"":"v",:2023-09-22T23:21:37.704770848Z""0name",,":""name"hostname:"crucible"":crucible",",""level"ip-10-150-1-55.us-west-2.compute.internal:"40level",:"30pid":4769}
{"msg":"[0] Starts reconcile loop","v":,"0time",:,"""name"time"2023-09-22T23:21:37.704814844Z::"""crucible",,2023-09-22T23:21:37.704817293Z"""hostname"level:,""":hostname"30ip-10-150-1-55.us-west-2.compute.internal":,""pid":4769ip-10-150-1-55.us-west-2.compute.internal","}pid"
:4769}
,{"Sep 22 23:21:37.704 INFO Downstairs has completed Negotiation, task: proc
{time""msg"Sep 22 23:21:37.704 INFO [1] f712941d-0577-43e4-8a2e-7814ec270c09 (512f09eb-7e91-421e-9fa9-f1bb0acbe6ae) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
"::""msg2023-09-22T23:21:37.704852322Z"[0] Starts reconcile loop",,""v"hostname":":0:,"""name":Sep 22 23:21:37.704 INFO [1] Transition from WaitActive to WaitQuorum
ip-10-150-1-55.us-west-2.compute.internal"",[0] client is_active_req TRUE, promote! session 9165874c-5197-4919-861e-2430574d8700""cruciblepid,""":,4769v"":}level
Sep 22 23:21:37.704 WARN [1] new RM replaced this: None
0"{:,30Sep 22 23:21:37.704 INFO Downstairs has completed Negotiation, task: proc
""msg"name"::""crucible","level":30,"[1] 8e88d9c1-84f3-4981-affe-0c1310d04226 (d738ea25-1afc-4d4c-9b80-99c2ada34ff7) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorumtime"":","v":2023-09-22T23:21:37.704920297Z"0,","name":"hostname":crucibleSep 22 23:21:37.704 INFO [1] Starts reconcile loop
",""level":30ip-10-150-1-55.us-west-2.compute.internal",,""time"pid"::"47692023-09-22T23:21:37.704933687Z"}
,"hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal"{,2023-09-22T23:21:37.704950472Z""pid","":hostname"msg"4769::""}
ip-10-150-1-55.us-west-2.compute.internal",[1] 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d (412fdf38-1b7f-416c-b371-8bd0b7f86f94) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum""pid",:"4769{v":}0
,""msg":"name":"{crucible","level":30"msg"[1] dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 (9165874c-5197-4919-861e-2430574d8700) WaitActive New New ds_transition to WaitActive":","v":0[1] Transition from WaitActive to WaitQuorum",",name"":v"":crucible0",",name"":"level"crucible":,","level"30time"::"{302023-09-22T23:21:37.705002119Z","hostname":""msg":"Sep 22 23:21:37.705 INFO [2] f712941d-0577-43e4-8a2e-7814ec270c09 (512f09eb-7e91-421e-9fa9-f1bb0acbe6ae) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
[0] Starts reconcile loop",ip-10-150-1-55.us-west-2.compute.internal,""v"time:":,"0",""2023-09-22T23:21:37.705021001Ztime"name",,"":"hostname:"""Sep 22 23:21:37.705 INFO [2] Transition from WaitActive to WaitQuorum
2023-09-22T23:21:37.705023806Z"crucible:"",pid"",ip-10-150-1-55.us-west-2.compute.internallevel"",:"":pid30":hostname"4769:Sep 22 23:21:37.705 WARN [2] new RM replaced this: None
"ip-10-150-1-55.us-west-2.compute.internal"},4769"
pid},"
:{"4769time":""}msg2023-09-22T23:21:37.705064238Z""{
:,""Sep 22 23:21:37.705 INFO [2] Starts reconcile loop
{[1] Transition from New to WaitActivehostname"""":,"msg"msg:"v"""::ip-10-150-1-55.us-west-2.compute.internal"[1] Transition from WaitActive to WaitQuorum""0,",[1] new RM replaced this: None",pid","":"4769namev""}:
"v:crucible0"{,""":,0msglevel""::",30""name":name"":"crucible"crucible",[1] 1c48f237-34b8-4484-b38b-7c6b80300cc8 (126b8002-1d8d-46c3-951e-15087f60a7c9) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","",levellevel"","v"time:"0:,":""2023-09-22T23:21:37.70513402Zname"":Sep 22 23:21:37.705 INFO Current flush_numbers [0..12]: [0, 0]
30Sep 22 23:21:37.705 INFO [0] 127.0.0.1:46295 task reports connection:true
40:,""cruciblehostname"":,""level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":{"",msg",time""::"""2023-09-22T23:21:37.705157125Z"time":,2023-09-22T23:21:37.705167378Z""[1] client is_active_req TRUE, promote! session 9165874c-5197-4919-861e-2430574d8700",",""Sep 22 23:21:37.705 INFO f712941d-0577-43e4-8a2e-7814ec270c09 WaitQuorum WaitQuorum WaitQuorum
2023-09-22T23:21:37.705162779Z"hostname,vhostname""""::"0:",hostname"ip-10-150-1-55.us-west-2.compute.internal""name:ip-10-150-1-55.us-west-2.compute.internal"",:,""""pid"pidcrucible""ip-10-150-1-55.us-west-2.compute.internal",:"4769:level}4769
"}Sep 22 23:21:37.705 INFO [0]R  flush_numbers: [0, 0]
,"
pid{:"30":msg"4769{:"}
Sep 22 23:21:37.705 INFO [0]R  generation: [0, 0]
"[1] Transition from WaitActive to WaitQuorum,""{msg"time,"":v"":"2023-09-22T23:21:37.705231984Z:"msg0,,""""Sep 22 23:21:37.705 INFO [0]R  dirty: [false, false]
:"[1] new RM replaced this: None"hostnamename""::"",[1] Starts reconcile loopcrucibleip-10-150-1-55.us-west-2.compute.internal"",,"""pidlevel"":"4769:Sep 22 23:21:37.705 INFO [1]R  flush_numbers: [0, 0]
,v"v""}::
3000,{","name""msg"Sep 22 23:21:37.705 INFO [1]R  generation: [0, 0]
name":,:""time"::"""crucible"crucible",2023-09-22T23:21:37.705280215Z,[2] dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 (9165874c-5197-4919-861e-2430574d8700) WaitActive WaitActive New ds_transition to WaitActive""",,"Sep 22 23:21:37.705 INFO [1]R  dirty: [false, false]
level"""v"hostname:0",:""level"nameip-10-150-1-55.us-west-2.compute.internal::""4030:,""pidcrucible"":4769,"level}"
:Sep 22 23:21:37.705 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.705 INFO [2]R  flush_numbers: [0, 0]
,","timetime"{30:""":msg"2023-09-22T23:21:37.705314865Z"":"2023-09-22T23:21:37.705315163ZSep 22 23:21:37.705 INFO [2]R  generation: [0, 0]
",,[1] new RM replaced this: None""time",:"","vhostname"2023-09-22T23:21:37.705335771Z"":",0":Sep 22 23:21:37.705 INFO [2]R  dirty: [false, false]
"hostname,"hostname"":name"ip-10-150-1-55.us-west-2.compute.internal":"ip-10-150-1-55.us-west-2.compute.internal"":,"",{cruciblepid"",:"4769"Sep 22 23:21:37.705 INFO Max found gen is 1
pid":}level
""ip-10-150-1-55.us-west-2.compute.internal{:4769msg"40msg":}"""
Sep 22 23:21:37.705 INFO Generation requested: 1 >= found:1
{,",[2] Transition from New to WaitActive""time,"":msgv:"""""2023-09-22T23:21:37.705387142Z::"0,"",hostnameSep 22 23:21:37.705 INFO Next flush: 1
pid[0] downstairs client at 127.0.0.1:39150 has UUID ea1b6b61-6132-4ae7-8fec-05be62de606d"":name"""[2] 8e88d9c1-84f3-4981-affe-0c1310d04226 (d738ea25-1afc-4d4c-9b80-99c2ada34ff7) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum":ip-10-150-1-55.us-west-2.compute.internal"":",,crucible""",pid""level,v":47690":,Sep 22 23:21:37.705 INFO All extents match
"4769name""}::}v30

"Sep 22 23:21:37.705 INFO No downstairs repair required
"{{,{"crucible"""",msgtime"""::":level[1] Starts reconcile loop"msgmsg""::"2023-09-22T23:21:37.705449062ZSep 22 23:21:37.705 INFO No initial repair work was required
"":[1] Starts reconcile loop30","","0,hostname,"":v"""[0] downstairs client at 127.0.0.1:54884 has UUID f2d164d9-9cbb-47eb-a838-b7f17a4d0411v""Sep 22 23:21:37.705 INFO Set Downstairs and Upstairs active
",ip-10-150-1-55.us-west-2.compute.internal",::"0,,""pidname":"4769:0",time""}crucible
":v"":"0name,name"""name2023-09-22T23:21:37.705483726Z:":"":crucible,,""crucible"{levelSep 22 23:21:37.705 INFO f712941d-0577-43e4-8a2e-7814ec270c09 is now active with session: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae
"hostname",:"":msg30"","""cruciblelevel"",:ip-10-150-1-55.us-west-2.compute.internal""level:",,30"Sep 22 23:21:37.705 INFO f712941d-0577-43e4-8a2e-7814ec270c09 Set Active after no repair
"pid"[2] client is_active_req TRUE, promote! session 9165874c-5197-4919-861e-2430574d8700"leveltime"":"4769::30}"
2023-09-22T23:21:37.705537337Z,"",v"""{Sep 22 23:21:37.705 INFO Notify all downstairs, region set compare is done.
",:msghostname:"0:,""time"":"time:","""2023-09-22T23:21:37.705549008Z:""[2] Transition from WaitActive to WaitQuorum"nameip-10-150-1-55.us-west-2.compute.internal"",:",30pid""":v"47692023-09-22T23:21:37.705561027Z,}"":
crucible"Sep 22 23:21:37.705 INFO Set check for repair
0,,{"hostname,"""hostname:name":""",crucible""ip-10-150-1-55.us-west-2.compute.internal:"time",:"",""ip-10-150-1-55.us-west-2.compute.internal""pid,"levelpid"":msglevelSep 22 23:21:37.705 INFO [1] 127.0.0.1:43385 task reports connection:true
":2023-09-22T23:21:37.705590544Z304769:""::30""}4769
}[0] 127.0.0.1:46967 task reports connection:true"
{,,""The guest has finished waiting for activation
,,Sep 22 23:21:37.705 INFO f712941d-0577-43e4-8a2e-7814ec270c09 Active Active Active
"{time"vtime"""msghostname"msg"":::"""::""":2023-09-22T23:21:37.705638044Z0",2023-09-22T23:21:37.705635209Z",name""hostname:"""crucible:"","ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internallevel"[2] 69c4c5e1-2fd5-4777-bf6a-fcd75618f55d (412fdf38-1b7f-416c-b371-8bd0b7f86f94) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f2d164d9-9cbb-47eb-a838-b7f17a4d0411, encrypted: true, database_read_version: 1, database_write_version: 1 }"",,"":Sep 22 23:21:37.705 INFO Set check for repair
""30pid","pidhostname"",,""::":4769,vv"""::time}"
:"ip-10-150-1-55.us-west-2.compute.internal"00,,""name",name:"":2023-09-22T23:21:37.70569801Z"crucible""",crucible"pid4769Sep 22 23:21:37.705 INFO [2] 127.0.0.1:64157 task reports connection:true
""level",,:4769":level30"":}30
hostname"}:"
ip-10-150-1-55.us-west-2.compute.internal"{,,,""time"time:"":"Sep 22 23:21:37.705 INFO f712941d-0577-43e4-8a2e-7814ec270c09 Active Active Active
"msg"2023-09-22T23:21:37.705743687Z:2023-09-22T23:21:37.705740717Z""""Sep 22 23:21:37.704 INFO UpstairsConnection { upstairs_id: dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9, session_id: 9165874c-5197-4919-861e-2430574d8700, gen: 1 } is now active (read-write)
pid,,{[2] new RM replaced this: None""":4769hostnamehostname""::"Sep 22 23:21:37.705 INFO Set check for repair
",}ip-10-150-1-55.us-west-2.compute.internal
"","""ip-10-150-1-55.us-west-2.compute.internalmsg{""pid,"":pid4769""}:
v":msg"::"0{4769,"""}msg
"name"{::1c48f237-34b8-4484-b38b-7c6b80300cc8 WaitQuorum WaitQuorum WaitActive""""msg",":crucible""7be22e7c-d55a-4790-a835-081c096012f4 WaitActive WaitActive WaitActive","v"[2] Transition from WaitActive to WaitQuorum,":,""vv""::00,,""name"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ea1b6b61-6132-4ae7-8fec-05be62de606d, encrypted: true, database_read_version: 1, database_write_version: 1 }0level:",name"""crucible"":name",:":"crucible""levelcrucible"":40,,30"",levellevel""Sep 22 23:21:37.705 INFO [0] received reconcile message
,::30",""vtimetime30"":",:"2023-09-22T23:21:37.705856283Z"",time"""hostname:":"":2023-09-22T23:21:37.705855201Z"02023-09-22T23:21:37.705871698Zip-10-150-1-55.us-west-2.compute.internal,,""",name,""pid""hostname:"4769":hostname:}"
ip-10-150-1-55.us-west-2.compute.internal"Sep 22 23:21:37.705 INFO [0] All repairs completed, exit
":,""{,crucible""ip-10-150-1-55.us-west-2.compute.internal"pidmsg""::"""time",[2] new RM replaced this: None4769"}pid"
"::,{4769""",msg2023-09-22T23:21:37.705878796Z"v:"":""0}Sep 22 23:21:37.705 INFO [0] Starts cmd_loop

,level"[1] downstairs client at 127.0.0.1:38174 has UUID 4b784146-9bfd-4506-a702-f66a5880a2a3,"""name{hostname,"":msg:"""cruciblev:""":,0":",level""30"name:"ip-10-150-1-55.us-west-2.compute.internal40:""[2] Starts reconcile loop",crucible",",""pid"v"level:":0:,,"30time"":name":"4769",2023-09-22T23:21:37.705965403Z"crucible}","",time,"":"hostname""
:2023-09-22T23:21:37.705978977Z""level,"time"::30ip-10-150-1-55.us-west-2.compute.internal"""hostname,"":pid""{:2023-09-22T23:21:37.70596311Zip-10-150-1-55.us-west-2.compute.internal4769"",}"
pid","{:Sep 22 23:21:37.706 INFO [1] received reconcile message
,""timemsg"4769"msg}:
"hostname:""{:""[2] Starts reconcile loop""Waiting for 1 more clients to be ready,msg""v"":"Sep 22 23:21:37.706 INFO [1] All repairs completed, exit
2023-09-22T23:21:37.706006075Z",:"0,"",vhostname""":"name"::[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4b784146-9bfd-4506-a702-f66a5880a2a3, encrypted: true, database_read_version: 1, database_write_version: 1 }":crucible""0ip-10-150-1-55.us-west-2.compute.internal",",,"ip-10-150-1-55.us-west-2.compute.internal"v",level,""pidname"""::030":Sep 22 23:21:37.706 INFO [1] Starts cmd_loop
"",cruciblepid"",name"":::47694769time""}:crucible"
{{2023-09-22T23:21:37.706079561Z"msg":"","hostname":"[0] downstairs client at 127.0.0.1:34554 has UUID a4638f9d-a209-4281-803d-046dfafa8f18","vip-10-150-1-55.us-west-2.compute.internal"":,0",pid"":name":"4769crucible"},"
levelSep 22 23:21:37.706 INFO [2] received reconcile message
"{:""30msg"msg"::""82efa4d5-f84d-4765-a042-3cbb8c544041 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30Sep 22 23:21:37.706 INFO [2] All repairs completed, exit
,"time":"2023-09-22T23:21:37.706168943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] downstairs client at 127.0.0.1:64794 has UUID 51f6c6d4-74e6-40e9-8073-39006e8120ae","v":0,"name":"crucible",""level":30,"",,"level""time:level""30:[0] 127.0.0.1:43936 task reports connection:true:""30,"v2023-09-22T23:21:37.7061952Z"":0,,""name"hostname:"":,"crucible"time"":","ip-10-150-1-55.us-west-2.compute.internal"level",:"30pid"2023-09-22T23:21:37.70621476Z":,4769"hostname"}:
",,""timetime""::"ip-10-150-1-55.us-west-2.compute.internal""2023-09-22T23:21:37.706223325Z",{",2023-09-22T23:21:37.706222303Z""time""msg:,,hostname"""hostname"":""pid2023-09-22T23:21:37.706232308Z:""ip-10-150-1-55.us-west-2.compute.internal:"","",ip-10-150-1-55.us-west-2.compute.internal""pid,":"pid:"47694769:[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a4638f9d-a209-4281-803d-046dfafa8f18, encrypted: true, database_read_version: 1, database_write_version: 1 }"hostname,"":}v"}"
4769
}{
"ip-10-150-1-55.us-west-2.compute.internalmsg":{{"0,,""":namepid""::{Sep 22 23:21:37.706 INFO Current flush_numbers [0..12]: [0, 0]
"msgSep 22 23:21:37.706 INFO [2] Starts cmd_loop
":4769}"
[0] downstairs client at 127.0.0.1:63497 has UUID f652e01a-1c15-4c9f-9d6f-06b9109bda2e","v"{:0",""cruciblename""msg"::",""crucible"level":69c4c5e1-2fd5-4777-bf6a-fcd75618f55d WaitQuorum WaitQuorum WaitQuorum"30,","level"v"::030,"name":"crucible",""level":30Set check for repair","v":0,"name":"crucible"msg,,"""level"time::,"":30time""":2023-09-22T23:21:37.706400458Z"",","2023-09-22T23:21:37.706410052Z"hostname",:""time"hostname":"ip-10-150-1-55.us-west-2.compute.internal:"","2023-09-22T23:21:37.70640568Zpid"ip-10-150-1-55.us-west-2.compute.internal:"4769"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 51f6c6d4-74e6-40e9-8073-39006e8120ae, encrypted: true, database_read_version: 1, database_write_version: 1 },},"
pid"""hostname,{:",4769"":"}msg
"v""The guest has finished waiting for activation
:time{":":msg""0:"",ip-10-150-1-55.us-west-2.compute.internal[0]R  flush_numbers: [0, 0]8f69534d-528b-4c23-b2c9-ce2dfe832ae1 WaitActive WaitActive WaitActive""""2023-09-22T23:21:37.706425301Z",,"name",v""v:"0:,0,,":""name"Sep 22 23:21:37.706 INFO Downstairs has completed Negotiation, task: proc
pidhostname":""crucible:"4769""name:"":},"cruciblecrucible""ip-10-150-1-55.us-west-2.compute.internal,"level",
"":,""levellevel""::3030{30pid":4769}"
msg":",,""timetime""::""{2023-09-22T23:21:37.706518303Z2023-09-22T23:21:37.706518172Z"",,""hostname"hostname""::"msg[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f652e01a-1c15-4c9f-9d6f-06b9109bda2e, encrypted: true, database_read_version: 1, database_write_version: 1 }"":ip-10-150-1-55.us-west-2.compute.internal""",,"ip-10-150-1-55.us-west-2.compute.internal,[1] 127.0.0.1:55221 task reports connection:true"time""",pid""pid:"4769:v}","
4769"}v:{
""msg"::"2023-09-22T23:21:37.706520126Z0:"{",0","[1] downstairs client at 127.0.0.1:59903 has UUID f9162fd6-864f-4c42-be11-73fe673d33ac"msg,name"name""":::""",crucible""[0]R  generation: [0, 0]""crucible"v,"":,hostname"0v,"":"0name,"":name""level:crucible":,:""30"crucible,"",""levellevel""::3030level"ip-10-150-1-55.us-west-2.compute.internal":,"30pid":4769}
,,""timetime""::"","2023-09-22T23:21:37.706610814Z2023-09-22T23:21:37.706610705Z"time"{,"":,hostname"":hostname""":"ip-10-150-1-55.us-west-2.compute.internal"msg",2023-09-22T23:21:37.706604995Zip-10-150-1-55.us-west-2.compute.internal""",,""pid,:"pid:"4769""}"time
:hostname"{":"4769msg"82efa4d5-f84d-4765-a042-3cbb8c544041 WaitActive WaitActive WaitActive}:""
:,ip-10-150-1-55.us-west-2.compute.internal"{","[0]R  dirty: [false, false]""pid"",""msgv"":"v":::2023-09-22T23:21:37.706614481Z04769,""0}name",[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f9162fd6-864f-4c42-be11-73fe673d33ac, encrypted: true, database_read_version: 1, database_write_version: 1 }""
:name,""v:"":"{0crucible",,""name",level""::"30"cruciblemsghostname"":crucible"":,""ip-10-150-1-55.us-west-2.compute.internal"",level",time"":,1c48f237-34b8-4484-b38b-7c6b80300cc8 WaitQuorum WaitQuorum WaitActive"30:Sep 22 23:21:37.706 INFO Current flush_numbers [0..12]: [0, 0]
"pid"",:level4769"",v""2023-09-22T23:21:37.706700295Z"time"},
:""0:hostname30,:"":""name"2023-09-22T23:21:37.706716406Z:"ip-10-150-1-55.us-west-2.compute.internal,"","hostname""pid:"":crucible"4769,"ip-10-150-1-55.us-west-2.compute.internal}level""
,:"30{pid":4769"}
msg,"{":""msg"time":"[1]R  flush_numbers: [0, 0]":","v":2023-09-22T23:21:37.706742126Z"8f69534d-528b-4c23-b2c9-ce2dfe832ae1 WaitActive WaitActive WaitActive0",",,name""v:"":"crucible0"hostname,,,""""namelevel"":":time"crucible::30"",""2023-09-22T23:21:37.706758366Z"level":ip-10-150-1-55.us-west-2.compute.internal,"",30hostname"":"pid":4769,"ip-10-150-1-55.us-west-2.compute.internal"time,""}:pid"
,"":The guest has finished waiting for activation
47692023-09-22T23:21:37.706784368Ztime""{}:,""
2023-09-22T23:21:37.706790859Zhostname""",:""msg"hostname{"ip-10-150-1-55.us-west-2.compute.internal:"",:"""msg"pidip-10-150-1-55.us-west-2.compute.internal"":,:4769"}pid
"":[2] downstairs client at 127.0.0.1:43168 has UUID 61a97a70-da7b-4cb5-91fa-9da487068e7c{4769"Waiting for 1 more clients to be ready"},"
msg"":{",v"[1]R  generation: [0, 0]""",msg""v:""v"::0:,0[2] downstairs client at 127.0.0.1:36502 has UUID 88162c01-9da1-4e8d-8218-aee73fa16e70,""0name,"":v""",namecrucible:"0",,"""level"name:name"30":::""crucible"",cruciblecrucible""",level"":,30,level""":timelevel30"":":302023-09-22T23:21:37.70686965Z",",hostname"":time"":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:21:37.706879606Z"",","pid":hostname"4769:"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4769{}
"msg":",{[1]R  dirty: [false, false]",,""msg"v:"":""0timetime",:""name"":2023-09-22T23:21:37.706884418Z":[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 88162c01-9da1-4e8d-8218-aee73fa16e70, encrypted: true, database_read_version: 1, database_write_version: 1 }"",,"crucible""v",:"0level,"":2023-09-22T23:21:37.706886626Z"30name"hostname":"":Sep 22 23:21:37.706 INFO Downstairs has completed Negotiation, task: proc
,crucible""time,,""level:"":""2023-09-22T23:21:37.706925189Z30"hostname":ip-10-150-1-55.us-west-2.compute.internal",",""hostname"pid":ip-10-150-1-55.us-west-2.compute.internal"":,,ip-10-150-1-55.us-west-2.compute.internal""time"4769,"":pid"pid""}:2023-09-22T23:21:37.706945185Z:"4769
4769},"
hostname":}{"{
"ip-10-150-1-55.us-west-2.compute.internal""msg,"":{"pid"msg"[2]R  flush_numbers: [0, 0]::"4769",}"
"Set check for repair"{v"msg":msg0",:"","name"v"":8f69534d-528b-4c23-b2c9-ce2dfe832ae1 WaitActive WaitActive WaitActive"":,crucible:"",v""0level:"0:,30",""name"name"::""crucible"crucible,"","level"level",:"30:[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 61a97a70-da7b-4cb5-91fa-9da487068e7c, encrypted: true, database_read_version: 1, database_write_version: 1 }30time"":","v":02023-09-22T23:21:37.707007957Z",","name"hostname":,:"""time"crucible:ip-10-150-1-55.us-west-2.compute.internal""","2023-09-22T23:21:37.707018347Z"pid",:"4769,hostname}"
:,""{"timeip-10-150-1-55.us-west-2.compute.internal""level",msg""pid:""::4769[2]R  generation: [0, 0]"}"":
,2023-09-22T23:21:37.707020307Z""30,"v"hostname"::0","name":"crucible"ip-10-150-1-55.us-west-2.compute.internal",","levelpid""::476930,"time"}:
"2023-09-22T23:21:37.707054156Z","hostname":"{,"time":"ip-10-150-1-55.us-west-2.compute.internal",""pid"msg":2023-09-22T23:21:37.707064806Z"":,"4769hostname":}"
[2] 1c48f237-34b8-4484-b38b-7c6b80300cc8 (126b8002-1d8d-46c3-951e-15087f60a7c9) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum"ip-10-150-1-55.us-west-2.compute.internal",,{""pidv""":msg"0::",4769"name"}:
82efa4d5-f84d-4765-a042-3cbb8c544041 WaitActive WaitActive WaitActive"","crucible{"v":,0","msglevel""":name""::"[2]R  dirty: [false, false]"30crucible",","v"level"::030,"name":"crucible","level":30}
{"msg":",",time",[0] 127.0.0.1:36647 task reports connection:truetime"""":,time"":":v""":2023-09-22T23:21:37.707127105Z0"2023-09-22T23:21:37.707118809Z2023-09-22T23:21:37.707123574Z"",,"",,hostnamename""::"""hostname"hostname":"ip-10-150-1-55.us-west-2.compute.internal"crucible",,"":levelpid""::304769""}
ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"{,""pid"pidmsg"""::47694769:,"}}"Max found gen is 1time""
,:
""{v"2023-09-22T23:21:37.707163305Z:"0,,""name"hostname:"{""msg"crucible::"""","msg"level[2] Transition from WaitActive to WaitQuorumip-10-150-1-55.us-west-2.compute.internal"":"30,:,""pid"v""::04769,"}name
,"[0] 82efa4d5-f84d-4765-a042-3cbb8c544041 (13fb2768-edc0-4158-a955-c73279bce790) WaitActive WaitActive WaitActive ds_transition to WaitQuorum""{time:,""msg""::""crucible""2023-09-22T23:21:37.707205363Z,v"8e88d9c1-84f3-4981-affe-0c1310d04226 WaitQuorum WaitQuorum WaitQuorum"""level,,"""hostnamev":::""0:30,ip-10-150-1-55.us-west-2.compute.internal0",,"""pidname"nameThe guest has finished waiting for activation
:",:4769""}time"
:crucible"""{msg",:""msg"":level""2023-09-22T23:21:37.707236443Z":7be22e7c-d55a-4790-a835-081c096012f4 WaitActive WaitActive WaitActive"Generation requested: 1 >= found:1",30,,""vv"":"0:,0"hostnamename,"":"name"":crucible:"""crucible,"","level"level:":ip-10-150-1-55.us-west-2.compute.internal3030",,""time"pid:"":4769}2023-09-22T23:21:37.707271534Z"
,",,"{hostname"time""":time""::""2023-09-22T23:21:37.70728467Zmsg"":,2023-09-22T23:21:37.707284991Z"""hostname,"":hostname""ip-10-150-1-55.us-west-2.compute.internal:"ip-10-150-1-55.us-west-2.compute.internal"",[2] new RM replaced this: None""ip-10-150-1-55.us-west-2.compute.internal,",,""pid":pid4769"":}4769
}v"
{:0,"{"name""msgmsg":"":":Next flush: 1""crucible,""[2] downstairs client at 127.0.0.1:34655 has UUID 10863a73-c1c6-434a-bb7b-15af38914b1bv"":,0",v"",name""::0",level""cruciblename"",:"":levelcrucible"":,pid"3040level""::476930}
,"time"{:","2023-09-22T23:21:37.707351675Ztime"",",":"time""hostname"::"2023-09-22T23:21:37.707355579Z""msg",":2023-09-22T23:21:37.707352447Z"hostnameip-10-150-1-55.us-west-2.compute.internal"",:,""""hostname"ip-10-150-1-55.us-west-2.compute.internalpid"",:"4769pid:}[0] Transition from WaitActive to WaitQuorum""
"ip-10-150-1-55.us-west-2.compute.internal",{:,4769""msg}"
:pid""{:"4769"All extents matchmsg"",:""vv""}::0
,0"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 10863a73-c1c6-434a-bb7b-15af38914b1b, encrypted: true, database_read_version: 1, database_write_version: 1 }name"",:,"{"cruciblev"",:"0",levelmsg"""":name30":":name""[2] Starts reconcile loop":crucible","crucible,""v"level"",:,:""time30"level:0"",:"302023-09-22T23:21:37.707423619Zname"":","crucible"hostname,"":time"",:"ip-10-150-1-55.us-west-2.compute.internal""level",2023-09-22T23:21:37.707437451Z""pid","::hostname30"4769:"}
ip-10-150-1-55.us-west-2.compute.internal",,"{"time"pid"":":msg"4769:"}2023-09-22T23:21:37.707442624Z
No downstairs repair required"",{,,"""msg"time"":v""hostname:"::7be22e7c-d55a-4790-a835-081c096012f4 WaitActive WaitActive WaitActive0"",""name2023-09-22T23:21:37.707455364Z"",:""vip-10-150-1-55.us-west-2.compute.internal",crucible":0",",hostname",""pidnamelevel""::"30"crucible"":":,4769"ip-10-150-1-55.us-west-2.compute.internal"},
,level""":time30"{:pid"":"4769msg2023-09-22T23:21:37.707495149Z""},
:,"""hostnametime""::""{[0] new RM replaced this: None","2023-09-22T23:21:37.70750698Zip-10-150-1-55.us-west-2.compute.internal""",,"msg""hostnamepid""::"4769:v}ip-10-150-1-55.us-west-2.compute.internal""
,"":{pid"0:"4769[2] 127.0.0.1:59938 task reports connection:true"msg}"
:,"{",v"No initial repair work was required"msg",:""v""name":"0::,""name[0] 7be22e7c-d55a-4790-a835-081c096012f4 (8fe9d3ab-892b-48e5-a656-2b4564d918d0) WaitActive WaitActive WaitActive ds_transition to WaitQuorum0",crucible","""vname":"0:,,""name:"":"level"crucible""crucible",crucible",level"",":"30level:"level:"30:4030,"time":","time"2023-09-22T23:21:37.707575943Z:"","hostname,"2023-09-22T23:21:37.707579173Z:""","time":ip-10-150-1-55.us-west-2.compute.internalhostname"",:""pid",:ip-10-150-1-55.us-west-2.compute.internal4769""},
""pid"time"::{4769"2023-09-22T23:21:37.707579909Z""}
msg,"{2023-09-22T23:21:37.707580708Z"""hostname",:msg"""::""hostname"Set Downstairs and Upstairs active":,[0] Transition from WaitActive to WaitQuorum""ip-10-150-1-55.us-west-2.compute.internal"v"",ip-10-150-1-55.us-west-2.compute.internal"",,:"0v":"pid",0"pid"name,:""4769::name"4769crucible"":,""}levelcrucible"":,30"}
level"
{,:"30time":""msg"{2023-09-22T23:21:37.707643947Z:"",""hostnamemsg""[0] Starts reconcile loop:"",:"",time""1c48f237-34b8-4484-b38b-7c6b80300cc8 WaitQuorum WaitQuorum WaitQuorum:ip-10-150-1-55.us-west-2.compute.internal"""v2023-09-22T23:21:37.707654244Z,"",""v"pid,:"0"hostname":,:4769:0}"
ip-10-150-1-55.us-west-2.compute.internal","{name""",msg""pid:""name"::4769:""69c4c5e1-2fd5-4777-bf6a-fcd75618f55d is now active with session: 412fdf38-1b7f-416c-b371-8bd0b7f86f94crucible""crucible,,}""v
"level:"",0{:"30",msg"":name""level"::"[0] new RM replaced this: None"crucible30,"",v"":level0":,"30name":"crucible","level":40,"time":",",time2023-09-22T23:21:37.707708237Z""",:"",time"time":2023-09-22T23:21:37.707716295Z""hostname",2023-09-22T23:21:37.707719192Z"":,hostname"""hostname:"""::ip-10-150-1-55.us-west-2.compute.internal""ip-10-150-1-55.us-west-2.compute.internal,"","ip-10-150-1-55.us-west-2.compute.internal"",pid""pid:"4769:pid2023-09-22T23:21:37.707713979Z"}4769
}"
{,":hostname{"4769msg"""}:msg"::""
[0] Starts reconcile loop69c4c5e1-2fd5-4777-bf6a-fcd75618f55d Set Active after no repair""",,""vv""::00,ip-10-150-1-55.us-west-2.compute.internal,""name"name:"{,"""pid"crucible::"4769""},crucible
""msg":"{,level""[0]R  flush_numbers: [0, 0]:level30"""msg":,30":v"":0,"name":"crucible",",level""time"::,""30[1] 82efa4d5-f84d-4765-a042-3cbb8c544041 (13fb2768-edc0-4158-a955-c73279bce790) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum"2023-09-22T23:21:37.707798357Ztime"",:,"""v"2023-09-22T23:21:37.707802482Zhostname"":,:""0hostname",:ip-10-150-1-55.us-west-2.compute.internal""","nameip-10-150-1-55.us-west-2.compute.internalpid"",:""pid4769"::},"
4769"crucible"}{,
"time"{msg":":""msg""2023-09-22T23:21:37.707813553Z"level":,:""30hostname"[1] 7be22e7c-d55a-4790-a835-081c096012f4 (8fe9d3ab-892b-48e5-a656-2b4564d918d0) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorumNotify all downstairs, region set compare is done."":,,""v"v:"0:,0","name"":name"":"crucible"ip-10-150-1-55.us-west-2.compute.internal",crucible"",,level""level:"30:"30pid,"":time"4769:"}
,,""2023-09-22T23:21:37.707850245Z"timetime""::""{,"2023-09-22T23:21:37.707864405Zhostname""2023-09-22T23:21:37.707865715Z"",,""msg"hostname"::hostname""":":ip-10-150-1-55.us-west-2.compute.internal""ip-10-150-1-55.us-west-2.compute.internal,"ip-10-150-1-55.us-west-2.compute.internal",""pidpid""::47694769,[0]R  generation: [0, 0]"}}

{,"{"pid"msgmsg":"":v"""4769:Set check for repair""},:
0[1] Transition from WaitActive to WaitQuorum""v",,":v{""0:name"0,,:""name""name""::""cruciblemsg"cruciblecrucible"",,"""level:The guest has finished waiting for activation
"level:",:30"30"level":[1] Transition from WaitActive to WaitQuorum30","v":0,"name":"crucible","level":30,,""timetime""::""2023-09-22T23:21:37.707949009Z2023-09-22T23:21:37.707947973Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47694769}}

{,,""{msg""time":msg"""::time"""[1] 127.0.0.1:39320 task reports connection:true[1] new RM replaced this: None"":,",""vv":2023-09-22T23:21:37.707951902Z2023-09-22T23:21:37.707958074Z""0",:"0,,name""name:"":"crucible",hostname"crucible:""",,"""levellevelhostname"""::40:ip-10-150-1-55.us-west-2.compute.internal"",30"pid":ip-10-150-1-55.us-west-2.compute.internal"4769,"pid":}4769
,,""}timetime""::""{
2023-09-22T23:21:37.708009295Z2023-09-22T23:21:37.708006646Z"""{,,""hostname"hostname:"":msg"":ip-10-150-1-55.us-west-2.compute.internal""msg",ip-10-150-1-55.us-west-2.compute.internal""pid,"""pid:"4769:}:"4769
}[1] new RM replaced this: None
{[0]R  dirty: [false, false]""{,"""msgmsg""::v",""":v"[1] Starts reconcile loop0"69c4c5e1-2fd5-4777-bf6a-fcd75618f55d Active Active Active,",,":v""v:"0:,0"0name",,""name""name"::"namecrucible"":,""level"crucible:"30,":"crucible""level",crucible"":,30level"":,40"level"time"::"302023-09-22T23:21:37.70807867Z","hostname,"":"time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:21:37.708086492Z",","pid,hostname""::"4769"}time"ip-10-150-1-55.us-west-2.compute.internal
":","{pid":2023-09-22T23:21:37.708088881Z""4769msg,,"}"
:time"{":"""msg":"hostname2023-09-22T23:21:37.708092577Z""Set check for repair:",,"[2] 7be22e7c-d55a-4790-a835-081c096012f4 (8fe9d3ab-892b-48e5-a656-2b4564d918d0) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum"""hostname"v,"":v0"ip-10-150-1-55.us-west-2.compute.internal":,0",:name"","name:"":ip-10-150-1-55.us-west-2.compute.internal"crucible"",""pid,crucible""level,pid""""level:"30:::3047694769}
}
,,""{timetime""::"""{msg"2023-09-22T23:21:37.708149339Z2023-09-22T23:21:37.708147738Z"":,,""""hostnamehostname"":[1] Starts reconcile loop:"""msg",:"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,"v"pid"":"pid:"4769:[1]R  flush_numbers: [0, 0]"}4769
},0"{
"vmsg",:{""":name""[2] Transition from WaitActive to WaitQuorummsg"":",0",v[2] 127.0.0.1:57826 task reports connection:true"":":name"0,,""v:"name""::0",""cruciblename"":crucible","",level"crucible"crucible:"30",,level""level"":level:30"30:30,"time":"2023-09-22T23:21:37.708225132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4769"time"}:
","2023-09-22T23:21:37.708233486Z{"time","":"hostnamemsg""::"",2023-09-22T23:21:37.708232977Z""[2] new RM replaced this: Noneip-10-150-1-55.us-west-2.compute.internal"",,,""pidv""::0time",4769""name}hostname
"::"""{:2023-09-22T23:21:37.70823208Z""cruciblemsg","",:"""levelip-10-150-1-55.us-west-2.compute.internalhostname"69c4c5e1-2fd5-4777-bf6a-fcd75618f55d Active Active Active":"40,"":,"v"":pid0ip-10-150-1-55.us-west-2.compute.internal","",name:","":time""pid"crucible:4769"":}4769
},2023-09-22T23:21:37.708280233Z""level{
,"":"30hostname"msg"::"{"ip-10-150-1-55.us-west-2.compute.internal","pid"":msg",4769":}time
""[2] 82efa4d5-f84d-4765-a042-3cbb8c544041 (13fb2768-edc0-4158-a955-c73279bce790) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum:{[1]R  generation: [0, 0]"""",2023-09-22T23:21:37.70831015Zmsg"",",:""v"hostname[2] Starts reconcile loop""::,"0","vip-10-150-1-55.us-west-2.compute.internal"":,0",pid"":name4769"name"}:
":"crucible{"crucible,""msg":level""":Set check for repair30",,""level"v"v"::030,"","name:time":"":0crucible"",,"2023-09-22T23:21:37.708351496Z""levelname","":hostname:"30:""crucible","ip-10-150-1-55.us-west-2.compute.internal"level",":pid"30:,4769"}time"
,":time"":{"2023-09-22T23:21:37.708359596Z""2023-09-22T23:21:37.708370612Z"msg":,"","hostname"hostname":[0] 127.0.0.1:54884 task reports connection:true"":","vip-10-150-1-55.us-west-2.compute.internal"":,0",ip-10-150-1-55.us-west-2.compute.internal",pid","name:""4769:"pid""}:
crucible"4769time"{},
:"""msglevel""::"302023-09-22T23:21:37.70837662Z"{[0] received reconcile message",,""hostname"":msg"":"v":0[1]R  dirty: [false, false]ip-10-150-1-55.us-west-2.compute.internal,,"""nametime,"":"":","2023-09-22T23:21:37.708413934Z"crucible"v,,""""levelhostname""::30"pid"::0ip-10-150-1-55.us-west-2.compute.internal",4769,"}"
,pid""name":time"4769::{}"
2023-09-22T23:21:37.70843782Z"",{"msg"hostname"":":msg""":crucible"ip-10-150-1-55.us-west-2.compute.internal""[2] Transition from WaitActive to WaitQuorum,",","pid7be22e7c-d55a-4790-a835-081c096012f4 WaitQuorum WaitQuorum WaitQuorum"""vlevel:,""4769"v}:
":30{:00",,"msg""name:"":name":"[0] All repairs completed, exit""crucible"crucible,"",v,"":"0level,"":level"30name"::30"crucible",,""timelevel""::"30,"2023-09-22T23:21:37.708481212Z"time":","hostname":2023-09-22T23:21:37.708495279Z"",,""timeip-10-150-1-55.us-west-2.compute.internal",:"hostnametime":"":""",ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:21:37.708503597Z"",",""pid2023-09-22T23:21:37.708497252Z"hostname""::4769",}pid
ip-10-150-1-55.us-west-2.compute.internal"",{":pid""msg""::4769"4769}hostname[0]R  flush_numbers: [0, 0]
"",{":}"vmsg""::0"
[0] Starts cmd_loop,"""name,"":"v":crucible0",,""{levelname""::30"ip-10-150-1-55.us-west-2.compute.internal""crucible"msg,"",:""pidlevel"",::[2]R  flush_numbers: [0, 0]""304769time",:}""v"
2023-09-22T23:21:37.708559631Z,"":,time"":{"hostname0"2023-09-22T23:21:37.708571052Z":,"","hostname""ip-10-150-1-55.us-west-2.compute.internal:name"""msg,""ip-10-150-1-55.us-west-2.compute.internalpid"",:"4769pid"}::4769
}:
{""crucible""{[2] new RM replaced this: None,msg""msg:":""""[0]R  generation: [0, 0][1] received reconcile message"level"",,""v",:":v0"v:,"0name","":name""30:crucible"":,crucible""0,level"","level:"30name"::30"crucible","level":40,",time"":time"":","2023-09-22T23:21:37.70863222Z"2023-09-22T23:21:37.708634261Z"time,,"""hostnamehostname""::"",:""time"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid"":::476947692023-09-22T23:21:37.708626752Z"}}

{",{""2023-09-22T23:21:37.70863756Zmsg""msg:"":hostname""[0]R  dirty: [false, false]"[1] All repairs completed, exit"":,,""vv""::00,,""name"name:"""crucible:"",crucibleip-10-150-1-55.us-west-2.compute.internal,""""level",:"30,"levelpid"hostname"::"4769":30}
,ip-10-150-1-55.us-west-2.compute.internal"","timepid"{,""::time"""4769msg":2023-09-22T23:21:37.708692661Z"":},2023-09-22T23:21:37.708698902Z"""
hostname,"":[2]R  generation: [0, 0]""hostname{",ip-10-150-1-55.us-west-2.compute.internal:"",""vip-10-150-1-55.us-west-2.compute.internal""pid,msg""""pid"::47694769::0}}

{,"{""msg""[2] Starts reconcile loopname"":,":msg"""crucible"[1]R  flush_numbers: [0, 0]:""v"[1] Starts cmd_loop:,,"""0level,v,"":"0v,":"0:name,"":name"30crucible"":,"""name"crucible:""level",:"30crucible"level,"":level"30:30,"time":"2023-09-22T23:21:37.708775538Z",,","time""hostname:time"",:2023-09-22T23:21:37.708779368Z"""time,"":"":ip-10-150-1-55.us-west-2.compute.internalhostname"",:""2023-09-22T23:21:37.70877022Z"pid","ip-10-150-1-55.us-west-2.compute.internal""2023-09-22T23:21:37.708780156Z,:"4769"hostname",:pid}""
"hostname":{ip-10-150-1-55.us-west-2.compute.internal":,"""msg4769":pid"}"
[1]R  generation: [0, 0]":,{"4769ip-10-150-1-55.us-west-2.compute.internal""v",}"
:msg0",:""pid"name{[2] received reconcile message""::,""crucible4769"v,"}":msg
0","level""name:"30:":{crucible"","level"[2]R  dirty: [false, false]":"msg"30,:""v",:"0[0] 127.0.0.1:39150 task reports connection:true",time":",name"""v:"2023-09-22T23:21:37.708855901Z,"":time,0"",hostname:"""":2023-09-22T23:21:37.708863446Z""crucible"name,ip-10-150-1-55.us-west-2.compute.internal,"","""hostnamelevel":"pid:":"4769:"30}ip-10-150-1-55.us-west-2.compute.internal
"crucible,"{",pid"""msg"level"::"4769:30}[1]R  dirty: [false, false]
","v"{:0,""name"msg:"":,crucible""","time"level[2] All repairs completed, exit""::,30""v":0,2023-09-22T23:21:37.708893953Z"",name",:"""crucible"time",hostname",""level"time:"30::"":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:21:37.708916807Z,""pid",2023-09-22T23:21:37.708904356Z:",4769"hostname"":,"time}""ip-10-150-1-55.us-west-2.compute.internalhostname":"",
:"2023-09-22T23:21:37.70892917Z""pid",:"4769hostname"}:
"{ip-10-150-1-55.us-west-2.compute.internal","{ip-10-150-1-55.us-west-2.compute.internal"pid,"""pidmsg"":"4769::msg"4769":}"}
[2]R  flush_numbers: [0, 0]"Max found gen is 1"{,,"""msgvv"
:""::0","0{,name[2] Starts cmd_loop"""msg:"name",""::""crucible""v",:"0crucible"82efa4d5-f84d-4765-a042-3cbb8c544041 WaitQuorum WaitQuorum WaitQuorum",level",:,30""name"level""::v"30":0crucible",","namelevel,""":"time:"30:crucible"","level":2023-09-22T23:21:37.708999788Z30","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid",:"4769time":"}
2023-09-22T23:21:37.709012443Z",",{,time""""msg"hostname:""::[2]R  generation: [0, 0]"""time"ip-10-150-1-55.us-west-2.compute.internal",2023-09-22T23:21:37.709005603Z",v"""::0pid",:"4769","}name
"2023-09-22T23:21:37.709015964Zhostname":":"",crucible""ip-10-150-1-55.us-west-2.compute.internal,""hostnamelevel",":"30pid"::"4769}ip-10-150-1-55.us-west-2.compute.internal"
,"pid":4769,"}time"
:{"{2023-09-22T23:21:37.709058874Z"",msg"""msg"hostname:""::"Generation requested: 1 >= found:1""[0]R  flush_numbers: [0, 0]","ip-10-150-1-55.us-west-2.compute.internal",,v""":v"0:,0pid",:"4769name""}:
"name"crucible":",{"crucible"level",":msg"30":"level":30[2]R  dirty: [false, false]","v":0,"name":"crucible","level":30,",time"":"time":"2023-09-22T23:21:37.709112891Z"2023-09-22T23:21:37.709110892Z",",,""timehostname"hostname"":::"""ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:21:37.709119378Z"",",pid""ip-10-150-1-55.us-west-2.compute.internalhostname""::,""4769pid":}ip-10-150-1-55.us-west-2.compute.internal
"4769,"pid":}4769{
"}msg":"
{[0]R  generation: [0, 0]","v{"":"0msg,"msg:":"""Next flush: 1"Max found gen is 1name",:""",v"":v0crucible"",:"0name,,""":level"name":"crucible"30,":"level":crucible"30,"level":30,"time":"2023-09-22T23:21:37.709197976Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time,"",:""pidtime""::"2023-09-22T23:21:37.709202694Z"47692023-09-22T23:21:37.709201246Z",}"
,hostname"":"{hostname":""ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"",msg"pid"pid"":::4769"4769}
[0]R  dirty: [false, false]}"
{,"v":0","msg"{:name"":"crucible"","msg"Generation requested: 1 >= found:1":level"",":v"All extents match":300,",v"":name"0:","crucible"name",:""level"crucible:"30,",level"":time"30:"2023-09-22T23:21:37.70927582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",",pid"":time4769":"}
2023-09-22T23:21:37.709286221Z","hostname{":"",msg"ip-10-150-1-55.us-west-2.compute.internal"time"":,:""pid"":2023-09-22T23:21:37.709290435Z"4769[1]R  flush_numbers: [0, 0]",},
""v"hostname"{::"0","msgname"ip-10-150-1-55.us-west-2.compute.internal"",:""pid":Next flush: 1"4769,"v":}0
,"name":"crucible","level"{:30"msg":":No downstairs repair required"",crucible""v",:"0level,"",name""::Sep 22 23:21:37.709 INFO Current flush_numbers [0..12]: [0, 0]
"crucible"time30,"":"level":302023-09-22T23:21:37.709343947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"{2023-09-22T23:21:37.709364771Z",,""msg""time"hostname":::"""All extents match"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:21:37.709368092Z,"""v",,"":hostname":pid"0":,4769Sep 22 23:21:37.709 DEBG IO Write 1000 has deps []
}
ip-10-150-1-55.us-west-2.compute.internal"","name"pid"{::"4769crucible"",}"
msg"level:"":{30[1]R  generation: [0, 0]","v":"0msg":,""name":"No initial repair work was requiredcrucible"","level",:"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.709429357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4769"time":}"
2023-09-22T23:21:37.709439796Z",","time"hostname":{":"2023-09-22T23:21:37.709445361Z""ip-10-150-1-55.us-west-2.compute.internal"msg",:"",hostname""No downstairs repair required:"pid"":,4769"v"}:ip-10-150-1-55.us-west-2.compute.internal
"0,",pid"":name":"4769{crucible"},
""msg"level":":{30[1]R  dirty: [false, false]","v":"0msg,"":name"":"crucible"Set Downstairs and Upstairs active",","levelv""::300,"name":"crucible","level":,30"time":"2023-09-22T23:21:37.709501163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time,"":"time"{:2023-09-22T23:21:37.709513469Z"","2023-09-22T23:21:37.709518552Z"msg","":hostnameSep 22 23:21:37.709 INFO Downstairs has completed Negotiation, task: proc
":"hostname"":"ip-10-150-1-55.us-west-2.compute.internal","No initial repair work was requiredpid"ip-10-150-1-55.us-west-2.compute.internal"":,,4769""v"}:
0pid",:"4769name{":"}crucible"
,""msglevel""::{30""msg":"1c48f237-34b8-4484-b38b-7c6b80300cc8 is now active with session: 126b8002-1d8d-46c3-951e-15087f60a7c9"[2]R  flush_numbers: [0, 0]",",v":"0v",":name":"0crucible",","namelevel""::"30crucible",","time":"level":302023-09-22T23:21:37.709577578Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.709593493Z"{,"hostname,"""msg"time":"::""ip-10-150-1-55.us-west-2.compute.internal","pid2023-09-22T23:21:37.709597985Z"Set Downstairs and Upstairs active"",":,hostname"4769v"":}"
ip-10-150-1-55.us-west-2.compute.internal":,0","{pid"name"::4769""msg"}:crucible"
","level":30{1c48f237-34b8-4484-b38b-7c6b80300cc8 Set Active after no repair","v":"0msg,"":"name":"[2]R  generation: [0, 0]crucible"",","level"v"::300,"name":"crucible","level",:"30time":"2023-09-22T23:21:37.709653098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":","2023-09-22T23:21:37.709665982Z{time"":",""hostname"2023-09-22T23:21:37.709671284Z"msg"::"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal7be22e7c-d55a-4790-a835-081c096012f4 is now active with session: 8fe9d3ab-892b-48e5-a656-2b4564d918d0"4769",",}v"
:"0pid",:"4769{name":"}crucible"
","msg"level:"":{30Notify all downstairs, region set compare is done.",""v":msg0",:""name":"crucible"[2]R  dirty: [false, false]",",level"":v"30:0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.709725366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,,""time"time"::{""2023-09-22T23:21:37.709736835Z"2023-09-22T23:21:37.70974142Z"",msg":,"""hostnamehostname""::"7be22e7c-d55a-4790-a835-081c096012f4 Set Active after no repair"",ip-10-150-1-55.us-west-2.compute.internal""ip-10-150-1-55.us-west-2.compute.internalv","":pid,"":0pid",4769:"4769name"}:
"}crucible"
{,"level":30{"msg":"Set check for repair"",msg""v":":0Max found gen is 1","name",:""v"crucible":,"0level",:"30name":","crucible"time",:""level":302023-09-22T23:21:37.709802572Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"The guest has finished waiting for activation
,2023-09-22T23:21:37.709819717Z""{time",:"""hostname"msg2023-09-22T23:21:37.709825352Z"":,":""hostname"ip-10-150-1-55.us-west-2.compute.internal":","Notify all downstairs, region set compare is done.pid""ip-10-150-1-55.us-west-2.compute.internal",:,4769""vpid}"
"::04769,"{name":}"
crucible"","msg"level"::"30{[0] received reconcile message","v":"0msg",:""name":"crucible"Generation requested: 1 >= found:1,""level",:"30v":0,"name":"crucible",",level"":time30":"2023-09-22T23:21:37.70988159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.709893374Z"{,,""hostname":""time"msg":ip-10-150-1-55.us-west-2.compute.internal":"","pidSet check for repair"2023-09-22T23:21:37.709899856Z"":,,""4769hostname"v":}":
ip-10-150-1-55.us-west-2.compute.internal"0,",pid""{:name"4769:"crucible""},
"msg"level:"":{30[0] All repairs completed, exit","v":"0msg,"":name"":"Next flush: 1"crucible",,""v"level"::030,"name":"crucible",","time"level:"":302023-09-22T23:21:37.709955701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.709968556Z"{,"hostname":",""time"ip-10-150-1-55.us-west-2.compute.internal":,""msg"pid"::"47692023-09-22T23:21:37.709974898Z"},
[1] 127.0.0.1:38174 task reports connection:true"",hostname"":{v"":0,""ip-10-150-1-55.us-west-2.compute.internal"name",msg":"":pidcrucible"",":"4769[0] Starts cmd_loop"level":,30"}v"
:0,"name":"crucible",{"level":30"msg":"All extents match","v":,"0time,"":name":""crucible",2023-09-22T23:21:37.710023519Z""level",:"30hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid"2023-09-22T23:21:37.710035123Z":4769,"hostname"}:
"ip-10-150-1-55.us-west-2.compute.internal","{pid",:"4769"time"msg":}:
""2023-09-22T23:21:37.71005024Z"7be22e7c-d55a-4790-a835-081c096012f4 Active Active Active{",","hostnamev""::"0",msg"":"ip-10-150-1-55.us-west-2.compute.internal"name":,[1] received reconcile message""",crucible""pid,"v""::04769level",:}"
name"30:"crucible","level{":30"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:21:37.710108105Z"time":","hostname":2023-09-22T23:21:37.710111955Z"","hostname":ip-10-150-1-55.us-west-2.compute.internal"","pid":4769ip-10-150-1-55.us-west-2.compute.internal","}pid"
:,4769"time":"}{
2023-09-22T23:21:37.710122609Z""msg",:"{"hostname":"Set check for repair"","ip-10-150-1-55.us-west-2.compute.internal"vmsg,"""pid":::0",4769"name[1] All repairs completed, exit"}:
""crucible",,"{"v"level"::"30msg":"0,"name":"No initial repair work was required"crucible",,""v"level"::030,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.710181779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"},time"
":"time":"2023-09-22T23:21:37.710190251Z"{2023-09-22T23:21:37.710194368Z",","hostname"":hostname"msg""::""ip-10-150-1-55.us-west-2.compute.internal","pid":[2] 127.0.0.1:34655 task reports connection:true4769"ip-10-150-1-55.us-west-2.compute.internal",},"
v"":pid"0:,{"4769name":"}
"crucible"msg",:"{level"":"30[1] Starts cmd_loop"msg":","v":0Set Downstairs and Upstairs active",","namev""::0","cruciblename"":","crucible"level",:"30level":30,"time":"2023-09-22T23:21:37.710250342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"time"}:
,""time":"2023-09-22T23:21:37.710264802Z"{,"2023-09-22T23:21:37.710264162Z"hostname":",""hostnamemsg""ip-10-150-1-55.us-west-2.compute.internal"::"","pid":4769ip-10-150-1-55.us-west-2.compute.internal"}7be22e7c-d55a-4790-a835-081c096012f4 Active Active Active",
",pid""{v"::47690",}"msg"
:name"":"crucible"{,"level":3082efa4d5-f84d-4765-a042-3cbb8c544041 is now active with session: 13fb2768-edc0-4158-a955-c73279bce790"",msg"":v"":0[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.710324424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,,""name"time":{:""crucible"",2023-09-22T23:21:37.710336283Zmsg"":","level""Set check for repair":hostname,""30v":":0,"nameip-10-150-1-55.us-west-2.compute.internal"",:""pid":crucible"4769,"level"}:
30,"time":"{2023-09-22T23:21:37.710371105Z","hostname"":"msg":"ip-10-150-1-55.us-west-2.compute.internal"[2] All repairs completed, exit",",pid"":v"4769:0},
,""name"time":{:""crucible","2023-09-22T23:21:37.710385989Z""msglevel"",:"30:"hostname":"82efa4d5-f84d-4765-a042-3cbb8c544041 Set Active after no repair"ip-10-150-1-55.us-west-2.compute.internal",,""v"pid"::04769,"name":"}
crucible",",level"":time"30{:"2023-09-22T23:21:37.710415773Z""msg",:""hostname":"[0] received reconcile message",ip-10-150-1-55.us-west-2.compute.internal"",v"":pid"0:,4769"name":"}crucible"
,,""level":time"{30:"2023-09-22T23:21:37.710435769Z""msg",:""hostname":"[2] Starts cmd_loop","v":ip-10-150-1-55.us-west-2.compute.internal"0,","pid"name"::"4769crucible","}level"
:,30"time":"{2023-09-22T23:21:37.710459405Z","hostname"":"msg":"ip-10-150-1-55.us-west-2.compute.internal","Notify all downstairs, region set compare is done."pid":,4769"v":}0
,",name"":{"time"crucible":"","2023-09-22T23:21:37.710480894Zlevel"":msg",:""30hostname"[0] All repairs completed, exit":","v":0ip-10-150-1-55.us-west-2.compute.internal",","namepid""::"4769crucible","}level"
:30,"time":"2023-09-22T23:21:37.710509326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.710524632Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","Set check for repair"pid":,4769"v":0},
"name":"crucible","level"{:30"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.710560637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"{2023-09-22T23:21:37.710569854Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":[1] 127.0.0.1:64794 task reports connection:true"4769,"v":}0
,"The guest has finished waiting for activation
{name"{:"crucible","level"":"30msg"msg":":"[1] received reconcile message","v":dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 WaitActive WaitActive WaitActive"0,"name,"":"v":crucible"0,","level"name":":crucible30",","time":level"":302023-09-22T23:21:37.710614978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.710630265Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal",,""82efa4d5-f84d-4765-a042-3cbb8c544041 Active Active Activetimepid""":,:4769""v"}:
02023-09-22T23:21:37.710634904Z","name":,""{crucible"hostname,"""level"msg"::":30"ip-10-150-1-55.us-west-2.compute.internal",[1] All repairs completed, exit""pid":,"4769v":0,"}name"
:"crucible","level":30,"time":"2023-09-22T23:21:37.710680316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"{2023-09-22T23:21:37.710695167Z","hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal"Set check for repair",",pid"":v"4769:0,"}name":
"crucible","level":30{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.710731624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769},
"time":"2023-09-22T23:21:37.710742347Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769[2] 127.0.0.1:43168 task reports connection:true",}"
v":0,"name":"{crucible","level":30"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.710781732Z","hostname":""ip-10-150-1-55.us-west-2.compute.internal",":pid",:"4769"time"crucible:"}"
,2023-09-22T23:21:37.710790331Z"",level""{:hostname":"30"ip-10-150-1-55.us-west-2.compute.internal"msg,":""pid":476982efa4d5-f84d-4765-a042-3cbb8c544041 Active Active Active"}
,"v":0,"name":"{crucible","level":30"msg":"[2] All repairs completed, exit",,""time"v"::"0,"name"2023-09-22T23:21:37.71082574Z":"crucible",","hostname"level"::"30,"ip-10-150-1-55.us-west-2.compute.internal"time",:""pid":47692023-09-22T23:21:37.71084244Z"},"
hostname":"ip-10-150-1-55.us-west-2.compute.internal",",{"pid"time:"4769:""msg"}:
"2023-09-22T23:21:37.710857975Z"[0]R  flush_numbers: [0, 0]","{,"hostname"v":msg"""::"0ip-10-150-1-55.us-west-2.compute.internal",,Set check for repair"""name,"pid":":crucible"4769",v"}"
level:"0:,30{"name":"crucible"","msg"level"::"30[2] Starts cmd_loop","v":0,"name":"crucible",,""time"level:"":302023-09-22T23:21:37.710910967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",",pid"":time"4769:"}
2023-09-22T23:21:37.71091745Z","{hostname":,"""msg":time""ip-10-150-1-55.us-west-2.compute.internal:"",[0]R  generation: [0, 0]""2023-09-22T23:21:37.710927771Z,pid"""v:"4769:,"0}hostname
",":name""{:"crucibleip-10-150-1-55.us-west-2.compute.internal""",msg,""level"pid""::"4769:30[0] received reconcile message}"
,"v":0,"name":"crucible",,""level"time"::30"2023-09-22T23:21:37.710976451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":","time[0]R  dirty: [false, false]"",:"{"v":0,"2023-09-22T23:21:37.710990928Z"name":""crucible"msg,","level"::""30hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769[0] 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 (4c909eb0-0138-456e-a20b-76c3a0a91bc8) WaitActive WaitActive WaitActive ds_transition to WaitQuorum"}
,","timev""::{0","name":"2023-09-22T23:21:37.71102085Z"crucible"",msg",""hostname:"level""::"[0] All repairs completed, exit"30,"ip-10-150-1-55.us-west-2.compute.internal"v,""pid"::04769,"}name":
"crucible","{level":30"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":","crucible"time,"":"level":302023-09-22T23:21:37.711048273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":,4769"time":"}
,2023-09-22T23:21:37.711061075Z""time",:""{hostname":"2023-09-22T23:21:37.711071294Z","hostname""msg"ip-10-150-1-55.us-west-2.compute.internal":",:""pid[0] Transition from WaitActive to WaitQuorum""ip-10-150-1-55.us-west-2.compute.internal:,""v,"":pid04769,"":name"4769}:
}"
crucible"{,"{level":30""msg":"msg":"[1]R  generation: [0, 0]"[0] Starts cmd_loop","v",":v"0:,"0name",":name"":"crucible"crucible",","level"level:"30:,"30time":"2023-09-22T23:21:37.711120467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"time}":"
,2023-09-22T23:21:37.711136988Z"","time"hostname"{Sep 22 23:21:37.711 DEBG up_ds_listen was notified
:":""2023-09-22T23:21:37.71113849Z"msg":","ip-10-150-1-55.us-west-2.compute.internal"hostname"[0] new RM replaced this: None,:"""pid":4769,ip-10-150-1-55.us-west-2.compute.internal"},
""pidv"{:"0":msg"4769,:"}"
Sep 22 23:21:37.711 DEBG up_ds_listen process 1000
[1]R  dirty: [false, false]"name{,"":v""":msg0",crucible":",name"":""level"crucible":,40"[1] received reconcile message"level":,30"v":0,"name":"crucible","level":30Sep 22 23:21:37.711 DEBG [A] ack job 1000:1, : downstairs
,"time":"2023-09-22T23:21:37.711215832Z",",hostname"":"time":","ip-10-150-1-55.us-west-2.compute.internal"time,""2023-09-22T23:21:37.711213701Z":"pid,"":2023-09-22T23:21:37.711221512Z4769hostname"}"
,:""{hostname":""ip-10-150-1-55.us-west-2.compute.internal"msg",:""pid"ip-10-150-1-55.us-west-2.compute.internal"[2]R  flush_numbers: [0, 0]:"4769,","pid}"v"
::04769,"name":"}{crucible
""msg",":"level"{:[0] Starts reconcile loop"30,"v"":Sep 22 23:21:37.711 DEBG up_ds_listen checked 1 jobs, back to waiting
msg0":","name":"[1] All repairs completed, exit",crucible"",,time""":level""v"::302023-09-22T23:21:37.711278643Z"0,,""hostname"name"::""crucible","level"ip-10-150-1-55.us-west-2.compute.internal":,"30pid":4769}
{,""msg":"time":"[2]R  generation: [0, 0]",2023-09-22T23:21:37.711297423Z""v":,0","hostname":name"":,""ip-10-150-1-55.us-west-2.compute.internal"time":crucible",,""pid"level"::476930}
,"time":"2023-09-22T23:21:37.711333363Z","{hostname":""ip-10-150-1-55.us-west-2.compute.internal"","2023-09-22T23:21:37.711307152Z"pid"msg:,4769""}hostname":
:""{ip-10-150-1-55.us-west-2.compute.internal"",msg"":pid"":[1] 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 (4c909eb0-0138-456e-a20b-76c3a0a91bc8) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum"[2]R  dirty: [false, false]4769,"",}"v"v"
::00,",name""{:name"":"crucible"",crucible"msg"":level"",:"[1] Starts cmd_loop"30level",:"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.711394753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time"{:"","msgtime""2023-09-22T23:21:37.711397693Z"::"","Max found gen is 1"hostname2023-09-22T23:21:37.711403109Z","":v"":,0",ip-10-150-1-55.us-west-2.compute.internalhostname"":name"":"","crucible"pid",ip-10-150-1-55.us-west-2.compute.internal"":level,"4769":pid"30:}4769
}
{,"time":"{"2023-09-22T23:21:37.711445059Z""msg",:"msg"hostname"":[1] Transition from WaitActive to WaitQuorum:""","[2] received reconcile messagevip-10-150-1-55.us-west-2.compute.internal""",,:""0pid,"":name":v""4769:crucible0",},""level"
name"::30"{crucible",""levelmsg""::"30Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,",time":""time":"2023-09-22T23:21:37.71149051Z",2023-09-22T23:21:37.711496357Z""time",,:"""hostname"hostname"::"2023-09-22T23:21:37.711503023Z"",ip-10-150-1-55.us-west-2.compute.internal"",ip-10-150-1-55.us-west-2.compute.internalhostname""pid""::4769","pid"}ip-10-150-1-55.us-west-2.compute.internal
":,4769"pid"}:{4769
}"
msg"{:"{"[1] new RM replaced this: None""msgmsg",:"":"v""Next flush: 1":[2] All repairs completed, exit",0","vv,"":"name0":,:""name"crucible:0",",""name"cruciblelevel""::,40""level":crucible30","level":30,"time":"2023-09-22T23:21:37.711583353Z",","time"hostname"::,"""time":"ip-10-150-1-55.us-west-2.compute.internal"2023-09-22T23:21:37.711581428Z",2023-09-22T23:21:37.711585492Z",""pid"hostname",":"hostname:"4769ip-10-150-1-55.us-west-2.compute.internal":,}"
ip-10-150-1-55.us-west-2.compute.internal"",pid"":{pid"4769:"4769}msg"
}:
{"All extents match","v"":{msg"0:,"""msg[1] Starts reconcile loop"name","":v":"":crucible"[2] Starts cmd_loop,""0,level""v","::300name":","crucible"name":,""cruciblelevel":"30,"level":30,"time":"2023-09-22T23:21:37.711660482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{,""time"msg",:"":"time"No downstairs repair required"2023-09-22T23:21:37.711667761Z:,""v"":,0",2023-09-22T23:21:37.71166939Z""hostname"name:","":"ip-10-150-1-55.us-west-2.compute.internalhostname""crucible",,""pid"::level"4769":30}ip-10-150-1-55.us-west-2.compute.internal"
,"pid":4769}
,{"time":""2023-09-22T23:21:37.711710776Z"msg":,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
[2] 8f69534d-528b-4c23-b2c9-ce2dfe832ae1 (4c909eb0-0138-456e-a20b-76c3a0a91bc8) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum"{,"v":"0msg",:""name":"crucibleNo initial repair work was required"",",v"":level"0:,"30name":"crucible","level":30,"time":"2023-09-22T23:21:37.711753832Z","hostname":","ip-10-150-1-55.us-west-2.compute.internaltime"":,""pid":47692023-09-22T23:21:37.711751098Z"}
,"hostname":"{"ip-10-150-1-55.us-west-2.compute.internalmsg"":,""pid":4769Set Downstairs and Upstairs active"},
"v":0,"name":"crucible"{,"level":30"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level",:"30time":"2023-09-22T23:21:37.711795557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":","time":"2023-09-22T23:21:37.711807133Z"8e88d9c1-84f3-4981-affe-0c1310d04226 is now active with session: d738ea25-1afc-4d4c-9b80-99c2ada34ff7",","hostnamev""::0","name":"crucible"ip-10-150-1-55.us-west-2.compute.internal,"","level"pid"::304769}
{,"time":""msg":2023-09-22T23:21:37.711837434Z"","hostname":"[2] new RM replaced this: None","v":ip-10-150-1-55.us-west-2.compute.internal0",,""namepid""::"4769crucible"},
"level":40{"msg":"8e88d9c1-84f3-4981-affe-0c1310d04226 Set Active after no repair","v":0,"name":"crucible","level":30,"time,"":"time":"2023-09-22T23:21:37.711876839Z"2023-09-22T23:21:37.711866809Z",","hostname"hostname":":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",,""pidpid""::47694769}
}
{"msg":"{Notify all downstairs, region set compare is done.",""v"msg"::0","name":"[2] Starts reconcile loop"crucible","v":0,"name":",crucible"",level"":level30":30,"time":"2023-09-22T23:21:37.711933338Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid":2023-09-22T23:21:37.711934409Z"4769,"}
hostname":"{ip-10-150-1-55.us-west-2.compute.internal",""pid"msg"::4769"}Set check for repair"
,"v":0,"name":"{crucible","level":30"msg":"[0] 127.0.0.1:34554 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.711976614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":,""time":"[1] 127.0.0.1:40751 task reports connection:true"2023-09-22T23:21:37.711985853Z","v",:"0,hostname""name"::""crucible","level":ip-10-150-1-55.us-west-2.compute.internal30","pid":4769}
,"{time":"2023-09-22T23:21:37.712014905Z"","msg"hostname"::""ip-10-150-1-55.us-west-2.compute.internal","8f69534d-528b-4c23-b2c9-ce2dfe832ae1 WaitQuorum WaitQuorum WaitQuorumpid"":4769,"v"}:
0,"name":{"crucible"","msg"level"::"308e88d9c1-84f3-4981-affe-0c1310d04226 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.712049847Z","time",:""hostname":"2023-09-22T23:21:37.712057162Z","ip-10-150-1-55.us-west-2.compute.internal"hostname",":"pid":4769ip-10-150-1-55.us-west-2.compute.internal"},
"pid":4769}
{{""msg"msg"::""Set check for repair"[0]R  flush_numbers: [0, 0]",",v"":v"0:,"0name",:""name":crucible"","crucible"level",:"30level":30,"time":"2023-09-22T23:21:37.712108278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,}"
time":"{2023-09-22T23:21:37.71210977Z"",msg"":"hostname":"[2] 127.0.0.1:42674 task reports connection:true"ip-10-150-1-55.us-west-2.compute.internal",",v"":pid"0:,4769"name":"}
crucible","level":30{"msg":"[0]R  generation: [0, 0]","v":0,"name":","crucible"time",:""level":302023-09-22T23:21:37.712150517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"8e88d9c1-84f3-4981-affe-0c1310d04226 Active Active Active",,""v":time"0:,""name":"crucible"2023-09-22T23:21:37.71216455Z,""level":30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.712189007Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid"[0]R  dirty: [false, false]":4769,"}v"
:0,"{name":"crucible""msg",:""level":30Set check for repair","v":0,"name":"crucible","level":30Sep 22 23:21:37.712 INFO current number of open files limit 65536 is already the maximum
,","time"time":":"2023-09-22T23:21:37.712226712Z"2023-09-22T23:21:37.712221296Z",",hostname":""hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid"":,"4769pid":}4769
}
{"msg":"{[0] received reconcile message",""v"msg"::0","name":"[1]R  flush_numbers: [0, 0]"crucible",,""v"level"::030,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.712273948Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"}
time":"{2023-09-22T23:21:37.712277937Z"","msg"hostname"::""[0] All repairs completed, exit"ip-10-150-1-55.us-west-2.compute.internal",",v"":pid"0:,"4769name":"}crucible"
,"level":30{"msg":"[1]R  generation: [0, 0]","v":0,",name"":"time":crucible"","level":2023-09-22T23:21:37.712316923Z"30,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Starts cmd_loop","v":0,","name"time"::""crucible","2023-09-22T23:21:37.712332942Zlevel"":30,"hostname":Sep 22 23:21:37.712 INFO Created new region file "/tmp/downstairs-qImf5Xje/region.json"
"ip-10-150-1-55.us-west-2.compute.internal","pid":,4769"time":}"
2023-09-22T23:21:37.712354866Z","hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid"":msg":4769"}
[1]R  dirty: [false, false]","v{":0","msg"name"::""crucible",[1] received reconcile message"","level"v"::300,"name":"crucible","level":30,"time":","time":2023-09-22T23:21:37.712399315Z"","hostname"2023-09-22T23:21:37.712395673Z:"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769ip-10-150-1-55.us-west-2.compute.internal"},
"pid":4769{}
"msg":"[1] All repairs completed, exit"{,"v":0,""name"msg"::""crucible","[2]R  flush_numbers: [0, 0]"level":,"30v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.712446684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"{2023-09-22T23:21:37.712451625Z""msg":","hostname[1] Starts cmd_loop"":,""v":0,"ip-10-150-1-55.us-west-2.compute.internalname"":,""crucible"pid,"":level"4769:30}
{,""time"msg"::""2023-09-22T23:21:37.712486074Z"[2]R  generation: [0, 0]",",hostname""v":":0,"nameip-10-150-1-55.us-west-2.compute.internal"",:""crucible"pid",":level4769":}
30Sep 22 23:21:37.712 DEBG Read      :1000 deps:[] res:true
{"msg":"[2] received reconcile message","v":0,"name":"crucible",",level":"30time":"2023-09-22T23:21:37.712517579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.71253251Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid[2]R  dirty: [false, false]"":4769,"v"}:
0,"name":"{crucible",""level"msg"::"30[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":","time"2023-09-22T23:21:37.712566154Z:"","hostname"2023-09-22T23:21:37.712572358Z":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4769,"pid}"
:4769}
{{""msg"msg"::""Max found gen is 1"[2] Starts cmd_loop",",v""v"::00,","name"name:"":"crucible"crucible",,""level"level"::3030,"time":"2023-09-22T23:21:37.712623824Z","hostname":","ip-10-150-1-55.us-west-2.compute.internal"time":,""pid":47692023-09-22T23:21:37.71262379Z"}
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.712670585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.712709119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.712747741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.71278635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.712825423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.71286326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"8f69534d-528b-4c23-b2c9-ce2dfe832ae1 is now active with session: 4c909eb0-0138-456e-a20b-76c3a0a91bc8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.712904969Z"Sep 22 23:21:37.712 DEBG Read      :1000 deps:[] res:true
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"8f69534d-528b-4c23-b2c9-ce2dfe832ae1 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.712946725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.712985703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.71302223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 127.0.0.1:59903 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.71306261Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"8f69534d-528b-4c23-b2c9-ce2dfe832ae1 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713108913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713148157Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 127.0.0.1:36502 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713188044Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"8f69534d-528b-4c23-b2c9-ce2dfe832ae1 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713227765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.71326463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713298636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713337138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769Sep 22 23:21:37.713 DEBG Read      :1000 deps:[] res:true
}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713378104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713415412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713450475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713488409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713527143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713565367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713597587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] downstairs client at 127.0.0.1:62341 has UUID edb26b00-76c7-4c82-b262-a6cf7181b4aa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713687092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: edb26b00-76c7-4c82-b262-a6cf7181b4aa, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"Sep 22 23:21:37.713 INFO Current flush_numbers [0..12]: [0, 0]
crucible","level":30,"time":"2023-09-22T23:21:37.713741738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713777023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] downstairs client at 127.0.0.1:51713 has UUID e80e1f8b-7f5a-43fe-b856-a2bd94bd70c3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713818749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e80e1f8b-7f5a-43fe-b856-a2bd94bd70c3, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713867418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.713910248Z"Sep 22 23:21:37.713 INFO Downstairs has completed Negotiation, task: proc
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.714 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.714 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.714 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.714 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.715 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.715 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
{"msg":"[0] dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 (9165874c-5197-4919-861e-2430574d8700) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.715420601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.715 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.715 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.716 INFO Opened existing region file "/tmp/downstairs-qImf5Xje/region.json"
Sep 22 23:21:37.716 INFO Database read version 1
Sep 22 23:21:37.716 INFO Database write version 1
Sep 22 23:21:37.716 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.716 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.717 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.717 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.717 INFO UUID: de5d1b92-54ba-463b-93eb-b2678e77643b
Sep 22 23:21:37.717 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.717 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.717 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.717 INFO Using address: 127.0.0.1:47135, task: main
Sep 22 23:21:37.717 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.718 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.718 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52307, task: repair
Sep 22 23:21:37.718 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52307, task: repair
Sep 22 23:21:37.718 INFO listening, local_addr: 127.0.0.1:52307, task: repair
Sep 22 23:21:37.718 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.718 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52307, task: repair
Sep 22 23:21:37.718 INFO Using repair address: 127.0.0.1:52307, task: main
Sep 22 23:21:37.718 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.718 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.718 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.718 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.719 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.719 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.719 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.719 DEBG Read      :1003 deps:[JobId(1001)] res:true
Sep 22 23:21:37.719 DEBG Read      :1003 deps:[JobId(1001)] res:true
Sep 22 23:21:37.720 DEBG Read      :1003 deps:[JobId(1001)] res:true
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720161453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:37.720211169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720257808Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 (9165874c-5197-4919-861e-2430574d8700) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720305417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720348056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":Sep 22 23:21:37.720 INFO listening on 127.0.0.1:0, task: main
"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:37.720394909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720436685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769Sep 22 23:21:37.720 WARN f712941d-0577-43e4-8a2e-7814ec270c09 request to replace downstairs 127.0.0.1:46295 with 127.0.0.1:47135
}
{"msg":"[2] dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 (9165874c-5197-4919-861e-2430574d8700) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:21:37.720 INFO f712941d-0577-43e4-8a2e-7814ec270c09 found old target: 127.0.0.1:46295 at 0
,"time":"2023-09-22T23:21:37.720489466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.720 INFO f712941d-0577-43e4-8a2e-7814ec270c09 replacing old: 127.0.0.1:46295 at 0
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:21:37.720 INFO [0] client skip 1 in process jobs because fault, : downstairs
,"time":"2023-09-22T23:21:37.72053247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40Sep 22 23:21:37.720 INFO [0] changed 1 jobs to fault skipped, : downstairs
,"time":"2023-09-22T23:21:37.720575984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Sep 22 23:21:37.720 INFO [0] f712941d-0577-43e4-8a2e-7814ec270c09 (512f09eb-7e91-421e-9fa9-f1bb0acbe6ae) Active Active Active ds_transition to Replacing
[2] Starts reconcile loop","v":0,"name":"crucible","level":30Sep 22 23:21:37.720 INFO [0] Transition from Active to Replacing
,"time":"2023-09-22T23:21:37.720621021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] 127.0.0.1:63497 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720662678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720704037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720743216Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720781079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720820073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.72086002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720901082Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720947342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
The guest has finished waiting for activation
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.720989068Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721028997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721068001Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721106958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721147185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721186971Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721226828Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721266239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721307292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721347653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 is now active with session: 9165874c-5197-4919-861e-2430574d8700","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721385992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721419481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"Sep 22 23:21:37.721 DEBG Write     :1000 deps:[] res:true
2023-09-22T23:21:37.721451796Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721489559Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 127.0.0.1:62341 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721521662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721553743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721584496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 127.0.0.1:51713 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721617898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"dbfe1e65-b9a0-4fe4-9ed9-1dc6d4d66db9 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721656905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721696197Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721735494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721778789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721816099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721856344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721900616Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721939754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.721979953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.722018296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.722058421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.722 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:37.722 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.723 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:37.723 WARN [0] f712941d-0577-43e4-8a2e-7814ec270c09 WARNING finish job 1000 when downstairs state:Replacing
Sep 22 23:21:37.723 WARN [0] Dropping already skipped job 1000, : downstairs
Sep 22 23:21:37.723 WARN [0] will exit pm_task, this downstairs Replacing
Sep 22 23:21:37.723 DEBG up_ds_listen was notified
Sep 22 23:21:37.723 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:21:37.724 ERRO 127.0.0.1:46295: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Replacing)), so we end too, looper: 0
Sep 22 23:21:37.724 INFO [0] f712941d-0577-43e4-8a2e-7814ec270c09 Gone missing, transition from Replacing to Replaced
Sep 22 23:21:37.724 INFO [0] f712941d-0577-43e4-8a2e-7814ec270c09 connection to 127.0.0.1:46295 closed, looper: 0
Sep 22 23:21:37.724 INFO [0] 127.0.0.1:46295 task reports connection:false
Sep 22 23:21:37.724 INFO f712941d-0577-43e4-8a2e-7814ec270c09 Replaced Active Active
Sep 22 23:21:37.724 INFO [0] 127.0.0.1:46295 task reports offline
Sep 22 23:21:37.724 WARN upstairs UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:21:37.724 WARN upstairs UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } was previously active, clearing, task: main
Sep 22 23:21:37.724 INFO connection (127.0.0.1:40703): all done
Sep 22 23:21:37.726 DEBG Write     :1001 deps:[JobId(1000)] res:true
test test::integration_test_guest_downstairs_unwritten_sparse_end ... ok
test test::integration_test_guest_downstairs_unwritten_span ... ok
Sep 22 23:21:37.728 DEBG Write     :1001 deps:[JobId(1000)] res:true
test test::integration_test_guest_downstairs_unwritten_sparse_1 ... ok
Sep 22 23:21:37.728 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.728 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.728 INFO current number of open files limit 65536 is already the maximum
test test::integration_test_guest_downstairs_unwritten_sparse_mid ... ok
Sep 22 23:21:37.728 INFO Created new region file "/tmp/downstairs-vIWrHsDo/region.json"
Sep 22 23:21:37.728 INFO Created new region file "/tmp/downstairs-GwPUeOkX/region.json"
Sep 22 23:21:37.728 INFO Created new region file "/tmp/downstairs-JXdGfx2t/region.json"
Sep 22 23:21:37.729 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.729 INFO Created new region file "/tmp/downstairs-7LycXkyy/region.json"
Sep 22 23:21:37.729 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.729 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.730 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.731 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.731 INFO Opened existing region file "/tmp/downstairs-GwPUeOkX/region.json"
Sep 22 23:21:37.731 INFO Database read version 1
Sep 22 23:21:37.731 INFO Database write version 1
Sep 22 23:21:37.732 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.732 INFO Opened existing region file "/tmp/downstairs-vIWrHsDo/region.json"
Sep 22 23:21:37.732 INFO Database read version 1
Sep 22 23:21:37.732 INFO Database write version 1
Sep 22 23:21:37.732 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.732 INFO Opened existing region file "/tmp/downstairs-JXdGfx2t/region.json"
Sep 22 23:21:37.732 INFO Database read version 1
Sep 22 23:21:37.732 INFO Database write version 1
Sep 22 23:21:37.732 INFO UUID: 4a9c9bb9-ba03-459d-9045-08a986b4daa9
Sep 22 23:21:37.732 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.733 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.733 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.733 INFO Using address: 127.0.0.1:44443, task: main
Sep 22 23:21:37.733 INFO UUID: 1108770c-6fd1-4459-a60b-614b367fa7d4
Sep 22 23:21:37.733 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.733 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.733 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.733 INFO Opened existing region file "/tmp/downstairs-7LycXkyy/region.json"
Sep 22 23:21:37.733 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.733 INFO Database read version 1
Sep 22 23:21:37.733 INFO Using address: 127.0.0.1:45992, task: main
Sep 22 23:21:37.733 INFO Database write version 1
Sep 22 23:21:37.733 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.733 INFO UUID: 96640be5-25af-4671-856f-955e28c0fd1e
Sep 22 23:21:37.733 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.733 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.733 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.733 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.733 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37497, task: repair
Sep 22 23:21:37.733 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.733 INFO Using address: 127.0.0.1:63880, task: main
Sep 22 23:21:37.733 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37497, task: repair
Sep 22 23:21:37.733 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58240, task: repair
Sep 22 23:21:37.733 INFO listening, local_addr: 127.0.0.1:37497, task: repair
Sep 22 23:21:37.733 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58240, task: repair
Sep 22 23:21:37.733 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.733 INFO listening, local_addr: 127.0.0.1:58240, task: repair
Sep 22 23:21:37.733 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58240, task: repair
Sep 22 23:21:37.733 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37497, task: repair
Sep 22 23:21:37.733 INFO Using repair address: 127.0.0.1:58240, task: main
Sep 22 23:21:37.733 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.733 INFO Using repair address: 127.0.0.1:37497, task: main
Sep 22 23:21:37.733 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.733 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.733 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:37.734 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61212, task: repair
Sep 22 23:21:37.734 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61212, task: repair
Sep 22 23:21:37.734 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.734 INFO listening, local_addr: 127.0.0.1:61212, task: repair
Sep 22 23:21:37.734 INFO UUID: c0634833-e1bc-4dc4-a207-c663e8474e16
Sep 22 23:21:37.734 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.734 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.734 INFO Created new region file "/tmp/downstairs-wFpZaG9t/region.json"
Sep 22 23:21:37.734 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.734 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.734 INFO Using address: 127.0.0.1:35164, task: main
Sep 22 23:21:37.734 INFO Created new region file "/tmp/downstairs-ccMKGrRU/region.json"
Sep 22 23:21:37.734 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61212, task: repair
Sep 22 23:21:37.734 INFO Using repair address: 127.0.0.1:61212, task: main
Sep 22 23:21:37.734 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.734 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.734 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.734 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53445, task: repair
Sep 22 23:21:37.734 INFO Created new region file "/tmp/downstairs-URcgTXmZ/region.json"
Sep 22 23:21:37.734 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53445, task: repair
Sep 22 23:21:37.734 INFO listening, local_addr: 127.0.0.1:53445, task: repair
Sep 22 23:21:37.734 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53445, task: repair
Sep 22 23:21:37.734 INFO Using repair address: 127.0.0.1:53445, task: main
Sep 22 23:21:37.734 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.735 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.735 INFO Created new region file "/tmp/downstairs-DB2vwjG7/region.json"
Sep 22 23:21:37.735 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:37.736 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:37.736 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:37.736 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.736 INFO Opened existing region file "/tmp/downstairs-wFpZaG9t/region.json"
Sep 22 23:21:37.736 INFO Database read version 1
Sep 22 23:21:37.736 INFO Database write version 1
Sep 22 23:21:37.737 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.737 INFO Opened existing region file "/tmp/downstairs-ccMKGrRU/region.json"
Sep 22 23:21:37.737 INFO Database read version 1
Sep 22 23:21:37.737 INFO Database write version 1
Sep 22 23:21:37.737 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.737 INFO Opened existing region file "/tmp/downstairs-URcgTXmZ/region.json"
Sep 22 23:21:37.737 INFO Database read version 1
Sep 22 23:21:37.737 INFO Database write version 1
Sep 22 23:21:37.738 INFO UUID: 5b712e05-efd4-443b-8e3d-e394f3713c4b
Sep 22 23:21:37.738 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.738 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.738 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.738 INFO Using address: 127.0.0.1:57990, task: main
Sep 22 23:21:37.738 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.738 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58749, task: repair
Sep 22 23:21:37.738 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58749, task: repair
Sep 22 23:21:37.738 INFO listening, local_addr: 127.0.0.1:58749, task: repair
Sep 22 23:21:37.738 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58749, task: repair
Sep 22 23:21:37.738 INFO Using repair address: 127.0.0.1:58749, task: main
Sep 22 23:21:37.738 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.739 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.739 INFO Opened existing region file "/tmp/downstairs-DB2vwjG7/region.json"
Sep 22 23:21:37.739 INFO Database read version 1
Sep 22 23:21:37.739 INFO Database write version 1
Sep 22 23:21:37.739 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.739 INFO Created new region file "/tmp/downstairs-qX3gXVLO/region.json"
Sep 22 23:21:37.739 INFO UUID: 2d0b2cb1-76dc-4a6c-9791-1330cfcd6cae
Sep 22 23:21:37.739 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.739 INFO UUID: 854ff323-c929-4b97-8826-1ca400654147
Sep 22 23:21:37.739 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.739 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.739 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.739 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.739 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.739 INFO Using address: 127.0.0.1:60947, task: main
Sep 22 23:21:37.739 INFO Using address: 127.0.0.1:57484, task: main
Sep 22 23:21:37.740 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.740 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.740 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52396, task: repair
Sep 22 23:21:37.740 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57392, task: repair
Sep 22 23:21:37.740 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57392, task: repair
Sep 22 23:21:37.740 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52396, task: repair
Sep 22 23:21:37.740 INFO listening, local_addr: 127.0.0.1:52396, task: repair
Sep 22 23:21:37.740 INFO listening, local_addr: 127.0.0.1:57392, task: repair
Sep 22 23:21:37.740 INFO UUID: 9aaa216b-96cd-4a39-8952-f14cf6369014
test test::integration_test_guest_downstairs_unwritten_span_2 ... ok
Sep 22 23:21:37.740 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.740 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52396, task: repair
Sep 22 23:21:37.740 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57392, task: repair
Sep 22 23:21:37.740 INFO Using repair address: 127.0.0.1:57392, task: main
Sep 22 23:21:37.740 INFO Using repair address: 127.0.0.1:52396, task: main
Sep 22 23:21:37.740 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.740 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.740 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.740 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.740 INFO Using address: 127.0.0.1:62068, task: main
Sep 22 23:21:37.741 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.741 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.741 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:54442, task: repair
Sep 22 23:21:37.741 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.741 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:54442, task: repair
Sep 22 23:21:37.741 INFO Created new region file "/tmp/downstairs-K70Wpcfy/region.json"
Sep 22 23:21:37.741 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.741 INFO listening, local_addr: 127.0.0.1:54442, task: repair
Sep 22 23:21:37.741 INFO Created new region file "/tmp/downstairs-wR25ZUX2/region.json"
Sep 22 23:21:37.741 INFO Created new region file "/tmp/downstairs-VaaXtxW5/region.json"
Sep 22 23:21:37.741 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:54442, task: repair
Sep 22 23:21:37.741 INFO Using repair address: 127.0.0.1:54442, task: main
Sep 22 23:21:37.741 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.742 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.742 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.742 INFO Opened existing region file "/tmp/downstairs-qX3gXVLO/region.json"
Sep 22 23:21:37.742 INFO Created new region file "/tmp/downstairs-nn01FdgV/region.json"
Sep 22 23:21:37.742 INFO Database read version 1
Sep 22 23:21:37.742 INFO Database write version 1
Sep 22 23:21:37.743 INFO UUID: 890345ba-e71f-4b71-b001-a87d38d96b67
Sep 22 23:21:37.743 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.743 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.743 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.743 INFO Using address: 127.0.0.1:35001, task: main
Sep 22 23:21:37.743 INFO Repair listens on 127.0.0.1:0, task: repair
test test::integration_test_guest_downstairs ... ok
Sep 22 23:21:37.743 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:54304, task: repair
Sep 22 23:21:37.743 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:54304, task: repair
Sep 22 23:21:37.744 INFO listening, local_addr: 127.0.0.1:54304, task: repair
Sep 22 23:21:37.744 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.744 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:54304, task: repair
Sep 22 23:21:37.744 INFO Using repair address: 127.0.0.1:54304, task: main
Sep 22 23:21:37.744 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.744 INFO Created new region file "/tmp/downstairs-8BOPdUGn/region.json"
Sep 22 23:21:37.745 INFO Upstairs starts
Sep 22 23:21:37.745 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.745 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.745 INFO Crucible stats registered with UUID: 7743f09f-f465-4fba-892d-457fc58d9abf
Sep 22 23:21:37.745 INFO Crucible 7743f09f-f465-4fba-892d-457fc58d9abf has session id: 9ebddc35-6adf-4dff-a4c5-9317a93426bd
The guest has requested activation
Sep 22 23:21:37.745 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.745 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.745 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.745 INFO Opened existing region file "/tmp/downstairs-K70Wpcfy/region.json"
Sep 22 23:21:37.745 INFO Database read version 1
Sep 22 23:21:37.745 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.745 INFO Database write version 1
Sep 22 23:21:37.745 INFO [0] connecting to 127.0.0.1:45992, looper: 0
Sep 22 23:21:37.745 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.745 INFO Opened existing region file "/tmp/downstairs-VaaXtxW5/region.json"
Sep 22 23:21:37.745 INFO Database read version 1
Sep 22 23:21:37.745 INFO Database write version 1
Sep 22 23:21:37.745 INFO [1] connecting to 127.0.0.1:57990, looper: 1
Sep 22 23:21:37.745 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.745 INFO Opened existing region file "/tmp/downstairs-wR25ZUX2/region.json"
Sep 22 23:21:37.745 INFO Database read version 1
Sep 22 23:21:37.745 INFO [2] connecting to 127.0.0.1:35001, looper: 2
Sep 22 23:21:37.745 INFO Database write version 1
Sep 22 23:21:37.745 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.745 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.745 INFO Flush timeout: 0.5
Sep 22 23:21:37.745 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.746 INFO 7743f09f-f465-4fba-892d-457fc58d9abf active request set
Sep 22 23:21:37.746 INFO Opened existing region file "/tmp/downstairs-nn01FdgV/region.json"
Sep 22 23:21:37.746 INFO Database read version 1
Sep 22 23:21:37.746 INFO Database write version 1
Sep 22 23:21:37.746 INFO accepted connection from 127.0.0.1:62688, task: main
Sep 22 23:21:37.746 INFO accepted connection from 127.0.0.1:38002, task: main
Sep 22 23:21:37.746 INFO accepted connection from 127.0.0.1:38970, task: main
Sep 22 23:21:37.746 INFO [0] 7743f09f-f465-4fba-892d-457fc58d9abf looper connected, looper: 0
Sep 22 23:21:37.746 INFO [0] Proc runs for 127.0.0.1:45992 in state New
Sep 22 23:21:37.746 INFO UUID: 9ea11016-ca01-4f8d-8100-108f556909ca
Sep 22 23:21:37.746 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.746 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.746 INFO [1] 7743f09f-f465-4fba-892d-457fc58d9abf looper connected, looper: 1
Sep 22 23:21:37.746 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.746 INFO Using address: 127.0.0.1:54777, task: main
Sep 22 23:21:37.746 INFO [1] Proc runs for 127.0.0.1:57990 in state New
Sep 22 23:21:37.746 INFO [2] 7743f09f-f465-4fba-892d-457fc58d9abf looper connected, looper: 2
Sep 22 23:21:37.746 INFO [2] Proc runs for 127.0.0.1:35001 in state New
Sep 22 23:21:37.746 INFO UUID: 62fc2f22-4d19-4964-a68a-fe053dd165a7
Sep 22 23:21:37.747 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.747 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.747 INFO UUID: 69d03179-d586-4376-83f6-52e7dee79d82
Sep 22 23:21:37.747 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.747 INFO UUID: a32c7e82-e087-4524-b977-a5b003758c5f
Sep 22 23:21:37.747 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.747 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.747 INFO Connection request from 7743f09f-f465-4fba-892d-457fc58d9abf with version 4, task: proc
Sep 22 23:21:37.747 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.747 INFO Using address: 127.0.0.1:51595, task: main
Sep 22 23:21:37.747 INFO upstairs UpstairsConnection { upstairs_id: 7743f09f-f465-4fba-892d-457fc58d9abf, session_id: 3175f1fa-0abb-4faa-9a83-0df20993aaa9, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.747 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58414, task: repair
Sep 22 23:21:37.747 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.747 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58414, task: repair
Sep 22 23:21:37.747 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.747 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.747 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.747 INFO Using address: 127.0.0.1:42987, task: main
Sep 22 23:21:37.747 INFO listening, local_addr: 127.0.0.1:58414, task: repair
Sep 22 23:21:37.747 INFO Using address: 127.0.0.1:35178, task: main
Sep 22 23:21:37.747 INFO Connection request from 7743f09f-f465-4fba-892d-457fc58d9abf with version 4, task: proc
Sep 22 23:21:37.747 INFO upstairs UpstairsConnection { upstairs_id: 7743f09f-f465-4fba-892d-457fc58d9abf, session_id: 3175f1fa-0abb-4faa-9a83-0df20993aaa9, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.747 INFO Connection request from 7743f09f-f465-4fba-892d-457fc58d9abf with version 4, task: proc
Sep 22 23:21:37.747 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58414, task: repair
Sep 22 23:21:37.747 INFO upstairs UpstairsConnection { upstairs_id: 7743f09f-f465-4fba-892d-457fc58d9abf, session_id: 3175f1fa-0abb-4faa-9a83-0df20993aaa9, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.747 INFO Using repair address: 127.0.0.1:58414, task: main
Sep 22 23:21:37.747 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.747 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.747 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.747 INFO Opened existing region file "/tmp/downstairs-8BOPdUGn/region.json"
Sep 22 23:21:37.747 INFO Database read version 1
Sep 22 23:21:37.747 INFO Database write version 1
Sep 22 23:21:37.747 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.747 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.747 INFO [0] 7743f09f-f465-4fba-892d-457fc58d9abf (3175f1fa-0abb-4faa-9a83-0df20993aaa9) New New New ds_transition to WaitActive
Sep 22 23:21:37.747 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35253, task: repair
Sep 22 23:21:37.747 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.747 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35253, task: repair
Sep 22 23:21:37.747 INFO [0] client is_active_req TRUE, promote! session 3175f1fa-0abb-4faa-9a83-0df20993aaa9
Sep 22 23:21:37.747 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61848, task: repair
Sep 22 23:21:37.747 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42081, task: repair
Sep 22 23:21:37.747 INFO listening, local_addr: 127.0.0.1:35253, task: repair
Sep 22 23:21:37.747 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61848, task: repair
Sep 22 23:21:37.747 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42081, task: repair
Sep 22 23:21:37.747 INFO listening, local_addr: 127.0.0.1:61848, task: repair
Sep 22 23:21:37.747 INFO listening, local_addr: 127.0.0.1:42081, task: repair
Sep 22 23:21:37.747 INFO [1] 7743f09f-f465-4fba-892d-457fc58d9abf (3175f1fa-0abb-4faa-9a83-0df20993aaa9) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.747 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.747 INFO [1] client is_active_req TRUE, promote! session 3175f1fa-0abb-4faa-9a83-0df20993aaa9
Sep 22 23:21:37.747 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35253, task: repair
Sep 22 23:21:37.747 INFO [2] 7743f09f-f465-4fba-892d-457fc58d9abf (3175f1fa-0abb-4faa-9a83-0df20993aaa9) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.747 INFO Using repair address: 127.0.0.1:35253, task: main
Sep 22 23:21:37.747 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.747 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.747 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61848, task: repair
Sep 22 23:21:37.747 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42081, task: repair
Sep 22 23:21:37.747 INFO [2] client is_active_req TRUE, promote! session 3175f1fa-0abb-4faa-9a83-0df20993aaa9
Sep 22 23:21:37.747 INFO Using repair address: 127.0.0.1:42081, task: main
Sep 22 23:21:37.747 INFO Using repair address: 127.0.0.1:61848, task: main
Sep 22 23:21:37.747 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.747 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.748 INFO UpstairsConnection { upstairs_id: 7743f09f-f465-4fba-892d-457fc58d9abf, session_id: 3175f1fa-0abb-4faa-9a83-0df20993aaa9, gen: 1 } is now active (read-write)
{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.748163817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0Sep 22 23:21:37.748 INFO UpstairsConnection { upstairs_id: 7743f09f-f465-4fba-892d-457fc58d9abf, session_id: 3175f1fa-0abb-4faa-9a83-0df20993aaa9, gen: 1 } is now active (read-write)
,"name":"crucible","level":30The guest has requested activation
,"time":"2023-09-22T23:21:37.7482556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.748292565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Sep 22 23:21:37.748 INFO listening on 127.0.0.1:0, task: main
Crucible stats registered with UUID: d1885708-091b-4eda-a722-18751f0474d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.748332484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible d1885708-091b-4eda-a722-18751f0474d0 has session id: 159bcbd6-d767-46d9-9279-e00f10ab7b2e","v":0Sep 22 23:21:37.748 INFO UpstairsConnection { upstairs_id: 7743f09f-f465-4fba-892d-457fc58d9abf, session_id: 3175f1fa-0abb-4faa-9a83-0df20993aaa9, gen: 1 } is now active (read-write)
Sep 22 23:21:37.748 INFO current number of open files limit 65536 is already the maximum
,Sep 22 23:21:37.748 INFO listening on 127.0.0.1:0, task: main
"name":"crucible","level":30,"time":"2023-09-22T23:21:37.748480875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.748 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.748 INFO UUID: 17366ba3-9d52-4dce-a4fd-d4657621aec9
Sep 22 23:21:37.748 INFO Created new region file "/tmp/downstairs-Ztyww1CP/region.json"
Sep 22 23:21:37.748 INFO Blocks per extent:5 Total Extents: 2
{"msg":"[0] connecting to 127.0.0.1:63880","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.748604148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
{Sep 22 23:21:37.748 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
"msg":"Upstairs starts","v":0,"name":"crucible","level":30Sep 22 23:21:37.748 INFO Upstairs <-> Downstairs Message Version: 4, task: main
,"time":"2023-09-22T23:21:37.748665328Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"Sep 22 23:21:37.748 INFO Using address: 127.0.0.1:61806, task: main
,"pid":4769}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.748739797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30The guest has requested activation
{,"time":"2023-09-22T23:21:37.748777338Z"","msghostname""::""Upstairs starts"ip-10-150-1-55.us-west-2.compute.internal,"",v"":pid":04769,"name}":
"crucible","level"{:30"msg":"Crucible stats registered with UUID: c8c6d6f5-1396-4842-801b-a86e7e53a2f6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.748809806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\nSep 22 23:21:37.748 INFO listening on 127.0.0.1:0, task: main
    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30The guest has requested activation
Sep 22 23:21:37.748 INFO listening on 127.0.0.1:0, task: main
,"time":"2023-09-22T23:21:37.748818979Z","hostname":","time":"ip-10-150-1-55.us-west-2.compute.internal","pid"2023-09-22T23:21:37.748886624Z:"4769,"hostname"}:"
ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{{"msg":""msg":"Sep 22 23:21:37.748 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.748 INFO listening on 127.0.0.1:0, task: main
Upstairs <-> Downstairs Message Version: 4Crucible c8c6d6f5-1396-4842-801b-a86e7e53a2f6 has session id: 12f55a26-57d6-442d-bf1e-32b5d4058f11"",,""vv""::00,,""namename""::""cruciblecrucible"",,""levellevel""::3030,,""timetime""::""2023-09-22T23:21:37.748993689Z2023-09-22T23:21:37.748993685Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:":47694769Sep 22 23:21:37.748 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.749 INFO listening on 127.0.0.1:0, task: main
{"msg":"Sep 22 23:21:37.749 INFO [0] downstairs client at 127.0.0.1:45992 has UUID 1108770c-6fd1-4459-a60b-614b367fa7d4
[1] connecting to 127.0.0.1:57484","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.749057175Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"Sep 22 23:21:37.749 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1108770c-6fd1-4459-a60b-614b367fa7d4, encrypted: true, database_read_version: 1, database_write_version: 1 }
1"}Sep 22 23:21:37.749 INFO listening on 127.0.0.1:0, task: main
}
{}
"Sep 22 23:21:37.749 INFO 7743f09f-f465-4fba-892d-457fc58d9abf WaitActive WaitActive WaitActive
{msg"Sep 22 23:21:37.749 INFO accepted connection from 127.0.0.1:34175, task: main
Sep 22 23:21:37.749 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57954, task: repair
":msg"":"[0] connecting to 127.0.0.1:44443","v":Crucible stats registered with UUID: c0eead19-4586-47fd-9018-7a0ce84810090",","name"v:"":0crucible",",name"":"level":crucible"30,"level":30,,""timetime""::""2023-09-22T23:21:37.749174306Z2023-09-22T23:21:37.749173045Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47694769,}"
looper":"0"}
{Sep 22 23:21:37.749 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57954, task: repair
{"msg"":msg"":"[1] connecting to 127.0.0.1:60947","v":0,Crucible c0eead19-4586-47fd-9018-7a0ce8481009 has session id: cbe2288b-c8af-400a-a089-f27ee961a317""name",:""v":crucible0",","name"level:"":30crucible","level":30
,,""timetime""::""2023-09-22T23:21:37.749223501ZSep 22 23:21:37.749 INFO [1] downstairs client at 127.0.0.1:57990 has UUID 5b712e05-efd4-443b-8e3d-e394f3713c4b
{"2023-09-22T23:21:37.749221765Z",","hostname"hostname:"":""msgip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"""pid,"":Sep 22 23:21:37.749 INFO listening, local_addr: 127.0.0.1:57954, task: repair
:"pid4769":4769}
,[2] connecting to 127.0.0.1:54777""{looper":"1","}msg"
:Sep 22 23:21:37.749 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 5b712e05-efd4-443b-8e3d-e394f3713c4b, encrypted: true, database_read_version: 1, database_write_version: 1 }
"v"":0[0] connecting to 127.0.0.1:35164",","namev""::0","cruciblename"":",crucible"","level"level"::3030,"time":"2023-09-22T23:21:37.749291015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0",}"
time":"2023-09-22T23:21:37.74929182Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal",[1] connecting to 127.0.0.1:62068""pid",:Sep 22 23:21:37.749 INFO 7743f09f-f465-4fba-892d-457fc58d9abf WaitActive WaitActive WaitActive
4769,"looper":"2"}
{"msg":"up_listen starts","v":0,"name":"crucible","level":Sep 22 23:21:37.749 INFO accepted connection from 127.0.0.1:42478, task: main
"v":0,"name":"crucible","level":3030Sep 22 23:21:37.749 INFO [2] downstairs client at 127.0.0.1:35001 has UUID 890345ba-e71f-4b71-b001-a87d38d96b67
,"time":"2023-09-22T23:21:37.749407381Z","hostname":"Sep 22 23:21:37.749 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57954, task: repair
ip-10-150-1-55.us-west-2.compute.internal",,""pid":time"4769:","2023-09-22T23:21:37.749415041Zlooper":Sep 22 23:21:37.749 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 890345ba-e71f-4b71-b001-a87d38d96b67, encrypted: true, database_read_version: 1, database_write_version: 1 }
{Sep 22 23:21:37.749 INFO Using repair address: 127.0.0.1:57954, task: main
"msg":"[2] connecting to 127.0.0.1:51595","v":0,"name":"crucible","level":30","hostname":"Sep 22 23:21:37.749 INFO accepted connection from 127.0.0.1:52416, task: main
Sep 22 23:21:37.749 INFO No SSL acceptor configured, task: main
,"Sep 22 23:21:37.749 INFO 7743f09f-f465-4fba-892d-457fc58d9abf WaitActive WaitActive WaitActive
time":"2023-09-22T23:21:37.74961871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"ip-10-150-1-55.us-west-2.compute.internallooper":""2","}pid"
:4769,"task":"{up_listen"}
"msg":"up_listen starts","v":0{,"name":"crucible","level"":msg":30"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.749714386Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"time":,""task":"2023-09-22T23:21:37.749723584Z"up_listen"},"
hostname":"Sep 22 23:21:37.749 INFO accepted connection from 127.0.0.1:40428, task: main
ip-10-150-1-55.us-west-2.compute.internal",{"pid":4769}"
msg":"Wait for all three downstairs to come online"{,"v":0,""name"msg"::""crucible"Flush timeout: 0.5",","level"v"::300,"name":"crucible","level":30,"time":","time":2023-09-22T23:21:37.749779533Z"","2023-09-22T23:21:37.749783441Z"hostname":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal4769","pid"}:
4769}
{{"msg":""Flush timeout: 0.5"msg":,""v":Sep 22 23:21:37.749 INFO accepted connection from 127.0.0.1:64171, task: main
d1885708-091b-4eda-a722-18751f0474d0 active request setSep 22 23:21:37.749 INFO Connection request from d1885708-091b-4eda-a722-18751f0474d0 with version 4, task: proc
0,Sep 22 23:21:37.749 INFO accepted connection from 127.0.0.1:48626, task: main
"","name"v"::"0crucible",","name"level"::"30crucible","level":30Sep 22 23:21:37.749 INFO upstairs UpstairsConnection { upstairs_id: d1885708-091b-4eda-a722-18751f0474d0, session_id: 39e89e28-13fd-421b-bfea-f7360e89a8c3, gen: 1 } connected, version 4, task: proc
,","time":time":""2023-09-22T23:21:37.749858407Z"2023-09-22T23:21:37.749860807Z",","hostname"hostname"::""ip-10-150-1-55.us-west-2.compute.internal"Sep 22 23:21:37.749 INFO Current flush_numbers [0..12]: [0, 0]
,"ip-10-150-1-55.us-west-2.compute.internalpid"":,"4769pid":4769}
}
{"{msg":""c8c6d6f5-1396-4842-801b-a86e7e53a2f6 active request set"msg":","v":0,"[1] d1885708-091b-4eda-a722-18751f0474d0 looper connected"name"Sep 22 23:21:37.749 INFO accepted connection from 127.0.0.1:49225, task: main
,:""crucible"v",:"0level",:"30name":"crucible","level":30,"time":"2023-09-22T23:21:37.749930615Z",","hostname"time"::""2023-09-22T23:21:37.749934051Z"ip-10-150-1-55.us-west-2.compute.internal",",pid"":hostname":4769"}
ip-10-150-1-55.us-west-2.compute.internal"Sep 22 23:21:37.749 INFO Connection request from d1885708-091b-4eda-a722-18751f0474d0 with version 4, task: proc
,{""pid"msg"::4769","looper":"1"[0] c8c6d6f5-1396-4842-801b-a86e7e53a2f6 looper connected"}
,{Sep 22 23:21:37.749 INFO current number of open files limit 65536 is already the maximum
""msg"Sep 22 23:21:37.749 INFO upstairs UpstairsConnection { upstairs_id: d1885708-091b-4eda-a722-18751f0474d0, session_id: 39e89e28-13fd-421b-bfea-f7360e89a8c3, gen: 1 } connected, version 4, task: proc
v:"":[1] Proc runs for 127.0.0.1:57484 in state New"0,,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.75002506Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
"name":"crucible"{,"level":"30msg":"[0] d1885708-091b-4eda-a722-18751f0474d0 looper connected","v":0,"name":"crucible","level":30","time":1"","2023-09-22T23:21:37.750059687Z"time"},"
hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
{:{""msg":"2023-09-22T23:21:37.75005447Z""[0] Proc runs for 127.0.0.1:63880 in state New"msg,"":,""v"hostname":[2] connecting to 127.0.0.1:42987"0:,",""namev""ip-10-150-1-55.us-west-2.compute.internal":,:0",""pid"crucible"name:"4769:,"",level":"30looper":"0"Sep 22 23:21:37.750 INFO Connection request from d1885708-091b-4eda-a722-18751f0474d0 with version 4, task: proc
Sep 22 23:21:37.750 INFO Created new region file "/tmp/downstairs-Ytj1TnsI/region.json"
crucible,",""level":30Sep 22 23:21:37.750 INFO upstairs UpstairsConnection { upstairs_id: d1885708-091b-4eda-a722-18751f0474d0, session_id: 39e89e28-13fd-421b-bfea-f7360e89a8c3, gen: 1 } connected, version 4, task: proc
,"time":"2023-09-22T23:21:37.750137091Z","hostname":"time":"2023-09-22T23:21:37.750113511Z"ip-10-150-1-55.us-west-2.compute.internal",",hostname"":"pid":4769ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"}looper":
"2"}
{{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":""msg":"2023-09-22T23:21:37.750197804Z","hostname":"[2] d1885708-091b-4eda-a722-18751f0474d0 looper connected","ip-10-150-1-55.us-west-2.compute.internalv"":,"0pid",:"4769name":"crucible",","task"level"::"30up_listen"}
Sep 22 23:21:37.750 INFO accepted connection from 127.0.0.1:59580, task: main
{"msg":","Wait for all three downstairs to come online"time":,""v":02023-09-22T23:21:37.750232226Z","name",:""crucible"hostname",}:""
ip-10-150-1-55.us-west-2.compute.internal"level",:"30pid":4769{,"looper":"2"}
"msg":"{[0] Proc runs for 127.0.0.1:44443 in state New",,"""timev"":msg"0:,"":"name"2023-09-22T23:21:37.750264958Z":"[2] Proc runs for 127.0.0.1:54777 in state New"crucible",,,"""hostname"level":v"::30"0,"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible",","pid"level"::476930}
{"msg":"Flush timeout: 0.5","v":0,,""name":"time"crucible":,,"""level"time2023-09-22T23:21:37.750299652Z:":","hostname":Sep 22 23:21:37.750 INFO Downstairs has completed Negotiation, task: proc
""ip-10-150-1-55.us-west-2.compute.internal","pid"2023-09-22T23:21:37.750306628Z":4769,"}hostname"
:{"msg":"[1] c8c6d6f5-1396-4842-801b-a86e7e53a2f6 looper connected","v":0,"name":"crucible","level":30"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.75045883Z30"{,"hostname":"ip-10-150-1-55.us-west-2.compute.internal"","msg":"pid":4769,"looper":"1"}
[1] d1885708-091b-4eda-a722-18751f0474d0 (39e89e28-13fd-421b-bfea-f7360e89a8c3) New New New ds_transition to WaitActive","v":0{,"name":",crucible"",""leveltime"msg"::""":302023-09-22T23:21:37.750483293Z"[1] Proc runs for 127.0.0.1:60947 in state New",","hostname"v"::"0,"name":"ip-10-150-1-55.us-west-2.compute.internal"crucible",","pid"level"::476930}
,"time":"2023-09-22T23:21:37.750511179Z"{,"hostname":""ip-10-150-1-55.us-west-2.compute.internal"msg,"":pid"":,4769"time"}:c0eead19-4586-47fd-9018-7a0ce8481009 active request set""
2023-09-22T23:21:37.750523708Z","v",:"{0hostname",:"""name":msg"ip-10-150-1-55.us-west-2.compute.internal"":crucible","",pid"[1] Transition from New to WaitActive":"4769,level"}v"
"::030,"name":"{crucible","level":30"msg":"[2] c8c6d6f5-1396-4842-801b-a86e7e53a2f6 looper connected","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:21:37.750575835Z"time":","hostname":2023-09-22T23:21:37.750579635Z"","hostname":ip-10-150-1-55.us-west-2.compute.internal"","pid":4769ip-10-150-1-55.us-west-2.compute.internal","}pid"
Sep 22 23:21:37.750 INFO Connection request from c8c6d6f5-1396-4842-801b-a86e7e53a2f6 with version 4, task: proc
Sep 22 23:21:37.750 INFO UpstairsConnection { upstairs_id: d1885708-091b-4eda-a722-18751f0474d0, session_id: 39e89e28-13fd-421b-bfea-f7360e89a8c3, gen: 1 } is now active (read-write)
Sep 22 23:21:37.750 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.750 INFO upstairs UpstairsConnection { upstairs_id: c8c6d6f5-1396-4842-801b-a86e7e53a2f6, session_id: ddd3f126-3dca-46a7-9147-f6bc461d9852, gen: 1 } connected, version 4, task: proc
:4769}
,"time":{"2023-09-22T23:21:37.750590231Z",""hostname":"msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769[1] client is_active_req TRUE, promote! session 39e89e28-13fd-421b-bfea-f7360e89a8c3",",looper"":"v"2":}0
,"name":"crucible","level":{30"msg":"[2] Proc runs for 127.0.0.1:51595 in state New","v":0,"name":"crucible","level":30Sep 22 23:21:37.750 INFO Connection request from c8c6d6f5-1396-4842-801b-a86e7e53a2f6 with version 4, task: proc
,"time":"2023-09-22T23:21:37.750738592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time"Sep 22 23:21:37.750 INFO upstairs UpstairsConnection { upstairs_id: c8c6d6f5-1396-4842-801b-a86e7e53a2f6, session_id: ddd3f126-3dca-46a7-9147-f6bc461d9852, gen: 1 } connected, version 4, task: proc
{:""msg":"2023-09-22T23:21:37.75075075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"[0] d1885708-091b-4eda-a722-18751f0474d0 (39e89e28-13fd-421b-bfea-f7360e89a8c3) New WaitActive New ds_transition to WaitActive",",pid"":v4769":0},
Sep 22 23:21:37.750 INFO UpstairsConnection { upstairs_id: d1885708-091b-4eda-a722-18751f0474d0, session_id: 39e89e28-13fd-421b-bfea-f7360e89a8c3, gen: 1 } is now active (read-write)
"name":"crucible","level":30,"time":"2023-09-22T23:21:37.75081321Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.750855452Z","hostname":"Sep 22 23:21:37.750 INFO accepted connection from 127.0.0.1:56561, task: main
Sep 22 23:21:37.750 INFO Downstairs has completed Negotiation, task: proc
ip-10-150-1-55.us-west-2.compute.internal"Sep 22 23:21:37.750 INFO Connection request from c8c6d6f5-1396-4842-801b-a86e7e53a2f6 with version 4, task: proc
,"pid":4769}
{"msg":"[0] client is_active_req TRUE, promote! session 39e89e28-13fd-421b-bfea-f7360e89a8c3","v":0,"name":"crucible","level":30Sep 22 23:21:37.750 INFO upstairs UpstairsConnection { upstairs_id: c8c6d6f5-1396-4842-801b-a86e7e53a2f6, session_id: ddd3f126-3dca-46a7-9147-f6bc461d9852, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.750 INFO UpstairsConnection { upstairs_id: d1885708-091b-4eda-a722-18751f0474d0, session_id: 39e89e28-13fd-421b-bfea-f7360e89a8c3, gen: 1 } is now active (read-write)
,"time":"2023-09-22T23:21:37.750909784Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] d1885708-091b-4eda-a722-18751f0474d0 (39e89e28-13fd-421b-bfea-f7360e89a8c3) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.750959983Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.75099863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] client is_active_req TRUE, promote! session 39e89e28-13fd-421b-bfea-f7360e89a8c3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.751034962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.751 INFO Current flush_numbers [0..12]: [0, 0]
{{"msg":""msg":"[0] c0eead19-4586-47fd-9018-7a0ce8481009 looper connected"[0] c8c6d6f5-1396-4842-801b-a86e7e53a2f6 (ddd3f126-3dca-46a7-9147-f6bc461d9852) New New New ds_transition to WaitActive,""v":0,,""name"v"::"0crucible",",name"":level"":crucible"30,"level":30,"time":"2023-09-22T23:21:37.751118431Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",","timepid""::4769","2023-09-22T23:21:37.751120827Z"looper":",0""}hostname":
"ip-10-150-1-55.us-west-2.compute.internal","{pid":4769"msg"}:"
[0] Proc runs for 127.0.0.1:35164 in state New","v":0,"name":"{crucible","level":30"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.751175415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"{2023-09-22T23:21:37.751184991Z",""hostname":"msg":"ip-10-150-1-55.us-west-2.compute.internal","pid"[1] c0eead19-4586-47fd-9018-7a0ce8481009 looper connected":,4769"v":0},
"name":"crucible","level":{30"msg":"[0] client is_active_req TRUE, promote! session ddd3f126-3dca-46a7-9147-f6bc461d9852","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.751224729Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1"}
,"time":"{2023-09-22T23:21:37.751238668Z","hostname"":"msg":"ip-10-150-1-55.us-west-2.compute.internal","[1] Proc runs for 127.0.0.1:62068 in state Newpid"":4769,"v":0},
"name":"crucible","level{":30"msg":"Sep 22 23:21:37.751 INFO Downstairs has completed Negotiation, task: proc
[1] c8c6d6f5-1396-4842-801b-a86e7e53a2f6 (ddd3f126-3dca-46a7-9147-f6bc461d9852) WaitActive New New ds_transition to WaitActive",",v"":time"0:,""name":"crucible"2023-09-22T23:21:37.751277694Z",",level"":hostname30":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"2023-09-22T23:21:37.751303972Z","Sep 22 23:21:37.751 INFO current number of open files limit 65536 is already the maximum
hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Transition from New to WaitActive","v":0,"Sep 22 23:21:37.751 INFO Opened existing region file "/tmp/downstairs-Ztyww1CP/region.json"
name":"crucible","level":30Sep 22 23:21:37.751 INFO Database read version 1
,"time":"2023-09-22T23:21:37.751361573Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.751 INFO Database write version 1
{"msg":"[1] client is_active_req TRUE, promote! session ddd3f126-3dca-46a7-9147-f6bc461d9852","v"Sep 22 23:21:37.751 INFO UpstairsConnection { upstairs_id: c8c6d6f5-1396-4842-801b-a86e7e53a2f6, session_id: ddd3f126-3dca-46a7-9147-f6bc461d9852, gen: 1 } is now active (read-write)
:0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.751414862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] c8c6d6f5-1396-4842-801b-a86e7e53a2f6 (ddd3f126-3dca-46a7-9147-f6bc461d9852) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30Sep 22 23:21:37.751 INFO [0] 7743f09f-f465-4fba-892d-457fc58d9abf (3175f1fa-0abb-4faa-9a83-0df20993aaa9) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
,"time":"2023-09-22T23:21:37.751462361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769Sep 22 23:21:37.751 INFO [0] Transition from WaitActive to WaitQuorum
{}
"msg":"{[1] downstairs client at 127.0.0.1:57484 has UUID 854ff323-c929-4b97-8826-1ca400654147""Sep 22 23:21:37.751 WARN [0] new RM replaced this: None
msg",:""v":0[2] Transition from New to WaitActive,""name,":""crucible"v",:"0level",":name":30"Sep 22 23:21:37.751 INFO UpstairsConnection { upstairs_id: c8c6d6f5-1396-4842-801b-a86e7e53a2f6, session_id: ddd3f126-3dca-46a7-9147-f6bc461d9852, gen: 1 } is now active (read-write)
crucible","level":30{"msg":"[2] c0eead19-4586-47fd-9018-7a0ce8481009 looper connected","v,"":time"0:","name":"2023-09-22T23:21:37.751537713Z,crucible""",,""time"level"hostname":::30""2023-09-22T23:21:37.751527448Zip-10-150-1-55.us-west-2.compute.internal"","pid":,4769"hostname"}:
"ip-10-150-1-55.us-west-2.compute.internal"{,"pid":4769"msg":"}
[2] client is_active_req TRUE, promote! session ddd3f126-3dca-46a7-9147-f6bc461d9852","v":0,"name":"{crucible",","time"level"::"30"msg"2023-09-22T23:21:37.751573313Z:"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,","looper"time:"":[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 854ff323-c929-4b97-8826-1ca400654147, encrypted: true, database_read_version: 1, database_write_version: 1 }"2"","}v"
2023-09-22T23:21:37.751591339Z":,0","hostname{"name"::"""cruciblemsg":ip-10-150-1-55.us-west-2.compute.internal""",",[2] Proc runs for 127.0.0.1:42987 in state Newpid"":"4769,level"}"
v"::300,"name":"crucible","level":30,"time":","time":2023-09-22T23:21:37.751631938Z"","hostname"2023-09-22T23:21:37.751628125Z:""Sep 22 23:21:37.751 INFO UpstairsConnection { upstairs_id: c8c6d6f5-1396-4842-801b-a86e7e53a2f6, session_id: ddd3f126-3dca-46a7-9147-f6bc461d9852, gen: 1 } is now active (read-write)
ip-10-150-1-55.us-west-2.compute.internal",,""hostname"pid"::"4769}
ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"d1885708-091b-4eda-a722-18751f0474d0 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.751688421Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] downstairs client at 127.0.0.1:63880 has UUID 96640be5-25af-4671-856f-955e28c0fd1e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.75172537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg"Sep 22 23:21:37.751 INFO [0] Starts reconcile loop
:"Sep 22 23:21:37.751 INFO Connection request from c0eead19-4586-47fd-9018-7a0ce8481009 with version 4, task: proc
[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 96640be5-25af-4671-856f-955e28c0fd1e, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.751782501Z","hostname"Sep 22 23:21:37.751 INFO upstairs UpstairsConnection { upstairs_id: c0eead19-4586-47fd-9018-7a0ce8481009, session_id: 64e4856d-2635-4307-a4f9-48bc00b7c13c, gen: 1 } connected, version 4, task: proc
:"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"d1885708-091b-4eda-a722-18751f0474d0 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.751827429Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] downstairs client at 127.0.0.1:54777 has UUID 9ea11016-ca01-4f8d-8100-108f556909ca","v":0,"name":"crucible","level":30Sep 22 23:21:37.751 INFO [1] 7743f09f-f465-4fba-892d-457fc58d9abf (3175f1fa-0abb-4faa-9a83-0df20993aaa9) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
,"time":"2023-09-22T23:21:37.751865165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.751 INFO [1] Transition from WaitActive to WaitQuorum
{"msg":"Sep 22 23:21:37.751 INFO Connection request from c0eead19-4586-47fd-9018-7a0ce8481009 with version 4, task: proc
[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9ea11016-ca01-4f8d-8100-108f556909ca, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible"Sep 22 23:21:37.751 WARN [1] new RM replaced this: None
,"level":30Sep 22 23:21:37.751 INFO upstairs UpstairsConnection { upstairs_id: c0eead19-4586-47fd-9018-7a0ce8481009, session_id: 64e4856d-2635-4307-a4f9-48bc00b7c13c, gen: 1 } connected, version 4, task: proc
,"time":"Sep 22 23:21:37.751 INFO Current flush_numbers [0..12]: [0, 0]
2023-09-22T23:21:37.751923663Z","hostname":"ip-10-150-1-55.us-west-2.compute.internalSep 22 23:21:37.751 INFO [1] Starts reconcile loop
","pid":4769}
{"msg":"d1885708-091b-4eda-a722-18751f0474d0 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.751975308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.752 INFO Connection request from c0eead19-4586-47fd-9018-7a0ce8481009 with version 4, task: proc
Sep 22 23:21:37.752 INFO [2] 7743f09f-f465-4fba-892d-457fc58d9abf (3175f1fa-0abb-4faa-9a83-0df20993aaa9) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.752 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.752 INFO upstairs UpstairsConnection { upstairs_id: c0eead19-4586-47fd-9018-7a0ce8481009, session_id: 64e4856d-2635-4307-a4f9-48bc00b7c13c, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.752 WARN [2] new RM replaced this: None
Sep 22 23:21:37.752 INFO [2] Starts reconcile loop
Sep 22 23:21:37.752 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.752 INFO [0] 127.0.0.1:45992 task reports connection:true
{"msg":"[0] downstairs client at 127.0.0.1:44443 has UUID 4a9c9bb9-ba03-459d-9045-08a986b4daa9","v":0,"name":"crucible","level":30Sep 22 23:21:37.752 INFO 7743f09f-f465-4fba-892d-457fc58d9abf WaitQuorum WaitQuorum WaitQuorum
,"time":"2023-09-22T23:21:37.752177989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Sep 22 23:21:37.752 INFO [0]R  flush_numbers: [0, 0]
[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4a9c9bb9-ba03-459d-9045-08a986b4daa9, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30Sep 22 23:21:37.752 INFO [0]R  generation: [0, 0]
,"time":"2023-09-22T23:21:37.752232846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Sep 22 23:21:37.752 INFO [0]R  dirty: [false, false]
c8c6d6f5-1396-4842-801b-a86e7e53a2f6 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,Sep 22 23:21:37.752 INFO [1]R  flush_numbers: [0, 0]
"time":"2023-09-22T23:21:37.752281139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.752 INFO [1]R  generation: [0, 0]
{"msg":"[1] downstairs client at 127.0.0.1:60947 has UUID 2d0b2cb1-76dc-4a6c-9791-1330cfcd6cae","v":0,"name":"crucible","level":Sep 22 23:21:37.752 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.752 INFO [1]R  dirty: [false, false]
30,"time":"Sep 22 23:21:37.752 INFO [2]R  flush_numbers: [0, 0]
2023-09-22T23:21:37.752336975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg"Sep 22 23:21:37.752 INFO [2]R  generation: [0, 0]
:"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2d0b2cb1-76dc-4a6c-9791-1330cfcd6cae, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","Sep 22 23:21:37.752 INFO [2]R  dirty: [false, false]
level":30,"Sep 22 23:21:37.752 INFO Max found gen is 1
time":"2023-09-22T23:21:37.75239918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"c8c6d6f5-1396-4842-801b-a86e7e53a2f6 WaitActive WaitActive WaitActive"Sep 22 23:21:37.752 INFO Generation requested: 1 >= found:1
,"v":0,"name":"crucible","level":30Sep 22 23:21:37.752 INFO UUID: 6c77c5f6-1f7d-446d-9f57-ac7866e05802
Sep 22 23:21:37.752 INFO Next flush: 1
,"time":"2023-09-22T23:21:37.752448709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769Sep 22 23:21:37.752 INFO Blocks per extent:5 Total Extents: 2
}
{"msg":"[2] downstairs client at 127.0.0.1:51595 has UUID 62fc2f22-4d19-4964-a68a-fe053dd165a7"Sep 22 23:21:37.752 INFO All extents match
,"v":0,"name":"crucible","level":30Sep 22 23:21:37.752 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.752 INFO No downstairs repair required
,"time":"2023-09-22T23:21:37.752506526Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":Sep 22 23:21:37.752 INFO No initial repair work was required
"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 62fc2f22-4d19-4964-a68a-fe053dd165a7, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","Sep 22 23:21:37.752 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
levelSep 22 23:21:37.752 INFO Set Downstairs and Upstairs active
"Sep 22 23:21:37.752 INFO Current flush_numbers [0..12]: [0, 0]
:30Sep 22 23:21:37.752 INFO UpstairsConnection { upstairs_id: c0eead19-4586-47fd-9018-7a0ce8481009, session_id: 64e4856d-2635-4307-a4f9-48bc00b7c13c, gen: 1 } is now active (read-write)
Sep 22 23:21:37.752 INFO Upstairs <-> Downstairs Message Version: 4, task: main
,"time":"2023-09-22T23:21:37.752578724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.752 INFO current number of open files limit 65536 is already the maximum
{Sep 22 23:21:37.752 INFO Using address: 127.0.0.1:54378, task: main
"Sep 22 23:21:37.752 INFO 7743f09f-f465-4fba-892d-457fc58d9abf is now active with session: 3175f1fa-0abb-4faa-9a83-0df20993aaa9
msg":"c8c6d6f5-1396-4842-801b-a86e7e53a2f6 WaitActive WaitActive WaitActive"Sep 22 23:21:37.752 INFO Opened existing region file "/tmp/downstairs-Ytj1TnsI/region.json"
,"v":0,"name":"crucible","level":30Sep 22 23:21:37.752 INFO Database read version 1
Sep 22 23:21:37.752 INFO 7743f09f-f465-4fba-892d-457fc58d9abf Set Active after no repair
,"time":"2023-09-22T23:21:37.752636451Z"Sep 22 23:21:37.752 INFO Database write version 1
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.752 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.752 INFO UpstairsConnection { upstairs_id: c0eead19-4586-47fd-9018-7a0ce8481009, session_id: 64e4856d-2635-4307-a4f9-48bc00b7c13c, gen: 1 } is now active (read-write)
Sep 22 23:21:37.752 INFO Set check for repair
Sep 22 23:21:37.752 INFO Current flush_numbers [0..12]: [0, 0]
{"msg":"[0] c0eead19-4586-47fd-9018-7a0ce8481009 (64e4856d-2635-4307-a4f9-48bc00b7c13c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.752737967Z"Sep 22 23:21:37.752 INFO Downstairs has completed Negotiation, task: proc
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769Sep 22 23:21:37.752 INFO [1] 127.0.0.1:57990 task reports connection:true
}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30Sep 22 23:21:37.752 INFO 7743f09f-f465-4fba-892d-457fc58d9abf Active Active Active
Sep 22 23:21:37.752 INFO UpstairsConnection { upstairs_id: c0eead19-4586-47fd-9018-7a0ce8481009, session_id: 64e4856d-2635-4307-a4f9-48bc00b7c13c, gen: 1 } is now active (read-write)
,"time":"2023-09-22T23:21:37.752794371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.752 INFO Set check for repair
{"msg":"[0] client is_active_req TRUE, promote! session 64e4856d-2635-4307-a4f9-48bc00b7c13c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.752840095Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Sep 22 23:21:37.752 INFO [2] 127.0.0.1:35001 task reports connection:true
[1] c0eead19-4586-47fd-9018-7a0ce8481009 (64e4856d-2635-4307-a4f9-48bc00b7c13c) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.752887563Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":Sep 22 23:21:37.752 INFO 7743f09f-f465-4fba-892d-457fc58d9abf Active Active Active
4769}
{"msg":"Sep 22 23:21:37.752 INFO Downstairs has completed Negotiation, task: proc
[1] Transition from New to WaitActive","v":0,"name":Sep 22 23:21:37.752 INFO Set check for repair
"crucible","level":30Sep 22 23:21:37.752 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.752 INFO Current flush_numbers [0..12]: [0, 0]
,"time":"2023-09-22T23:21:37.752945609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] client is_active_req TRUE, promote! session 64e4856d-2635-4307-a4f9-48bc00b7c13c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.752993779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.753 INFO [0] received reconcile message
{"msg":"[2] c0eead19-4586-47fd-9018-7a0ce8481009 (64e4856d-2635-4307-a4f9-48bc00b7c13c) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30Sep 22 23:21:37.753 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:55763, task: repair
Sep 22 23:21:37.753 INFO [0] All repairs completed, exit
,"time":"2023-09-22T23:21:37.753042836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Transition from New to WaitActive","v":0,"Sep 22 23:21:37.753 INFO [0] Starts cmd_loop
name":"crucible","level":30Sep 22 23:21:37.753 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:55763, task: repair
,"time":"2023-09-22T23:21:37.753093894Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.753 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.753 INFO [1] received reconcile message
{Sep 22 23:21:37.753 INFO [1] All repairs completed, exit
"msg":"[2] client is_active_req TRUE, promote! session 64e4856d-2635-4307-a4f9-48bc00b7c13c","v":0,"name":"{crucible","level":30"msg":"[1] d1885708-091b-4eda-a722-18751f0474d0 (39e89e28-13fd-421b-bfea-f7360e89a8c3) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:21:37.753 INFO listening, local_addr: 127.0.0.1:55763, task: repair
Sep 22 23:21:37.753 INFO [1] Starts cmd_loop
,"time":"2023-09-22T23:21:37.753244912Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.753319305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","Sep 22 23:21:37.753 INFO [2] received reconcile message
pid":4769}
{"msg":"[1] new RM replaced this: None","v":0Sep 22 23:21:37.753 INFO Current flush_numbers [0..12]: [0, 0]
,"name":"crucible","level":Sep 22 23:21:37.753 INFO [2] All repairs completed, exit
40,"time":"2023-09-22T23:21:37.753376976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"Sep 22 23:21:37.753 INFO [2] Starts cmd_loop
:4769}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30Sep 22 23:21:37.753 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:55763, task: repair
,"time":"2023-09-22T23:21:37.753425818Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] d1885708-091b-4eda-a722-18751f0474d0 (39e89e28-13fd-421b-bfea-f7360e89a8c3) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:21:37.753 INFO Using repair address: 127.0.0.1:55763, task: main
Sep 22 23:21:37.753 INFO UUID: e173e972-e8cf-416e-bc11-1d8d35f2e9ba
The guest has finished waiting for activation
,"time":"Sep 22 23:21:37.753 INFO No SSL acceptor configured, task: main
2023-09-22T23:21:37.753472561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.753 INFO Blocks per extent:5 Total Extents: 2
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.753520521Z","hostname":"Sep 22 23:21:37.753 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.753 DEBG Write     :1004 deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
ip-10-150-1-55.us-west-2.compute.internal","Sep 22 23:21:37.753 INFO current number of open files limit 65536 is already the maximum
pid":4769}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"Sep 22 23:21:37.753 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
2023-09-22T23:21:37.75357352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Starts reconcile loop"Sep 22 23:21:37.753 INFO Upstairs <-> Downstairs Message Version: 4, task: main
,"v":0,"name":"crucible","level":30Sep 22 23:21:37.753 INFO Using address: 127.0.0.1:36272, task: main
,"time":"2023-09-22T23:21:37.753615534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] d1885708-091b-4eda-a722-18751f0474d0 (39e89e28-13fd-421b-bfea-f7360e89a8c3) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:21:37.753 INFO current number of open files limit 65536 is already the maximum
,"time":"2023-09-22T23:21:37.753655857Z","Sep 22 23:21:37.753 INFO Created new region file "/tmp/downstairs-P4Ld1XOk/region.json"
hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.7537026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:37.753742872Z","hostname":"Sep 22 23:21:37.753 INFO Current flush_numbers [0..12]: [0, 0]
ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Starts reconcile loop","v":0,"name":"Sep 22 23:21:37.753 INFO Created new region file "/tmp/downstairs-LHGaEZWT/region.json"
crucible","level":30,"time":"2023-09-22T23:21:37.753800141Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 127.0.0.1:57484 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.753842875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"d1885708-091b-4eda-a722-18751f0474d0 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.753884453Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30The guest has finished waiting for activation
,"time":"2023-09-22T23:21:37.753923903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769Sep 22 23:21:37.753 INFO Repair listens on 127.0.0.1:0, task: repair
}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible","level":30Sep 22 23:21:37.753 INFO Downstairs has completed Negotiation, task: proc
,"time":"2023-09-22T23:21:37.753968803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754007603Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R  flush_numbers: [0, 0]"Sep 22 23:21:37.754 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:54512, task: repair
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754049684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Sep 22 23:21:37.754 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:54512, task: repair
[1]R  generation: [0, 0]","v":0,"name":"crucible","level":30Sep 22 23:21:37.754 DEBG Write     :1004 deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
,"time":"2023-09-22T23:21:37.754090455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R  dirty: [false, false]"Sep 22 23:21:37.754 INFO listening, local_addr: 127.0.0.1:54512, task: repair
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.75413138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754166891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"Sep 22 23:21:37.754 INFO Current flush_numbers [0..12]: [0, 0]
,"pid":4769}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754205861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754236968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754272442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{Sep 22 23:21:37.754 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:54512, task: repair
"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754314126Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.754 INFO Using repair address: 127.0.0.1:54512, task: main
{"msg":"Next flush: 1","v":0,"name":"crucible","level":30Sep 22 23:21:37.754 INFO No SSL acceptor configured, task: main
,"time":"2023-09-22T23:21:37.754351886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"All extents match","v":0Sep 22 23:21:37.754 INFO Downstairs has completed Negotiation, task: proc
,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.75439481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754431759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754467438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30The guest has finished waiting for activation
,"time":"Sep 22 23:21:37.754 INFO current number of open files limit 65536 is already the maximum
2023-09-22T23:21:37.754509977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"d1885708-091b-4eda-a722-18751f0474d0 is now active with session: 39e89e28-13fd-421b-bfea-f7360e89a8c3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754556337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"d1885708-091b-4eda-a722-18751f0474d0 Set Active after no repair","v":0,"name":"crucible","level":30Sep 22 23:21:37.754 INFO Current flush_numbers [0..12]: [0, 0]
,"time":"2023-09-22T23:21:37.754595431Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30Sep 22 23:21:37.754 INFO Created new region file "/tmp/downstairs-6LMZkwp5/region.json"
,"time":"2023-09-22T23:21:37.754634111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754669035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal"Sep 22 23:21:37.754 DEBG Write     :1004 deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
,"pid":4769}
{"msg":"[0] 127.0.0.1:63880 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754713537Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"d1885708-091b-4eda-a722-18751f0474d0 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754754396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair"Sep 22 23:21:37.754 INFO Downstairs has completed Negotiation, task: proc
,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754798315Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 127.0.0.1:54777 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754834837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"d1885708-091b-4eda-a722-18751f0474d0 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754868304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754898078Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754930975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.754966984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.755004601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30{"msg":","time":"2023-09-22T23:21:37.755041166Z"[0] c8c6d6f5-1396-4842-801b-a86e7e53a2f6 (ddd3f126-3dca-46a7-9147-f6bc461d9852) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","hostname":","v":0ip-10-150-1-55.us-west-2.compute.internal",","name"pid":":crucible"4769,"level"}:
30{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":","2023-09-22T23:21:37.755071581Ztime"":","hostname2023-09-22T23:21:37.755082981Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"ip-10-150-1-55.us-west-2.compute.internal":,"4769pid":4769}}

{{"msg":""msg":[0] Transition from WaitActive to WaitQuorum"","v"[1] Starts cmd_loop","v":0,"name":"crucible","level":30:0,"name":"crucible","level":30,"time":","2023-09-22T23:21:37.755133007Z"time":","hostname"2023-09-22T23:21:37.755144543Z":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4769,"pid":}4769
}
{"msg"{:"[2] received reconcile message"","msg"v":":0,"[0] new RM replaced this: None"name":","cruciblev"":,"0level",":name":30"crucible","level":40,"time":"2023-09-22T23:21:37.755207189Z",,""time"hostname"::""2023-09-22T23:21:37.755210302Z"ip-10-150-1-55.us-west-2.compute.internal",,""pid":hostname4769":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4769{}
"msg":"[2] All repairs completed, exit"{,"v":0","msgname""::""crucible","[0] Starts reconcile loop"level":,30"v":0,"name":"crucible","level":30,"time,"":"time":"2023-09-22T23:21:37.753219137Z"2023-09-22T23:21:37.755261866Z",","hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,,""pidpid"":"time:47694769":}"}

{2023-09-22T23:21:37.755267184Z",""hostnamemsg{"":":""[2] Starts cmd_loop"msg":","ip-10-150-1-55.us-west-2.compute.internal"v",:"0pid,[0] downstairs client at 127.0.0.1:35164 has UUID c0634833-e1bc-4dc4-a207-c663e8474e16""name","":v"":crucible0",,""name"level:"":30crucible","level":30:4769}
,"time":,""time"{:2023-09-22T23:21:37.755308734Z"","2023-09-22T23:21:37.755310934Z"","msg"hostname":":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
hostname"[1] c8c6d6f5-1396-4842-801b-a86e7e53a2f6 (ddd3f126-3dca-46a7-9147-f6bc461d9852) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum:""{,"v":0ip-10-150-1-55.us-west-2.compute.internal"",msg,"":pid""":name"4769:"}crucible"
,"level":30[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c0634833-e1bc-4dc4-a207-c663e8474e16, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.755351007Z",","hostname"time"::""2023-09-22T23:21:37.755357317Z"ip-10-150-1-55.us-west-2.compute.internal",,""hostname"pid:"":4769}ip-10-150-1-55.us-west-2.compute.internal
","pid":4769{}
"msg":"{[1] Transition from WaitActive to WaitQuorum",""msg":"v":0,"namec0eead19-4586-47fd-9018-7a0ce8481009 WaitActive WaitActive WaitActive"":,""v":0crucible,""name":","crucible"level",":level"30:30,"time":"2023-09-22T23:21:37.755408558Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal":,""pid":4769}
2023-09-22T23:21:37.755408251Z",{"hostname"":"msg":"ip-10-150-1-55.us-west-2.compute.internal","[1] downstairs client at 127.0.0.1:62068 has UUID 9aaa216b-96cd-4a39-8952-f14cf6369014pid"":,"4769v":0,"}name"
:"crucible","level":30{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:37.75544471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,}"
time":"2023-09-22T23:21:37.755456041Z"{,"hostname":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9aaa216b-96cd-4a39-8952-f14cf6369014, encrypted: true, database_read_version: 1, database_write_version: 1 }"","msg"v:"":0,"[1] Starts reconcile loopname"":","vcrucible"":0,","level"name:":"30crucible","level":30,"time,"":"time":"2023-09-22T23:21:37.755544244Z"2023-09-22T23:21:37.755546376Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal",ip-10-150-1-55.us-west-2.compute.internal""pid,"":pid4769":4769}
}
{{"msg":""msg":"c0eead19-4586-47fd-9018-7a0ce8481009 WaitActive WaitActive WaitActive","v"[2] c8c6d6f5-1396-4842-801b-a86e7e53a2f6 (ddd3f126-3dca-46a7-9147-f6bc461d9852) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible",":level"0:30,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.755584121Z",","hostname"time:"":"2023-09-22T23:21:37.755587871Zip-10-150-1-55.us-west-2.compute.internal"",","pid":hostname4769":"}
ip-10-150-1-55.us-west-2.compute.internal","pid":4769{}
"msg":"{[2] Transition from WaitActive to WaitQuorum"","msg"v:"":0,"name":"crucible","level"[2] downstairs client at 127.0.0.1:42987 has UUID 69d03179-d586-4376-83f6-52e7dee79d82:"30,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.755619243Z",",hostname"":"time":"ip-10-150-1-55.us-west-2.compute.internal2023-09-22T23:21:37.755623906Z"",","pid":hostname"4769:"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{",msg"":time"":"2023-09-22T23:21:37.755718762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 69d03179-d586-4376-83f6-52e7dee79d82, encrypted: true, database_read_version: 1, database_write_version: 1 }","v{":0,""namemsg""::""crucible","[2] Starts reconcile loop"level",:"v30":0,"name":"crucible","level":30The guest has finished waiting for activation
,","time"time:"":"2023-09-22T23:21:37.755757001Z"2023-09-22T23:21:37.755753565Z",","hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:"4769:4769}
}
{"{msg":""msg":"[0] 127.0.0.1:44443 task reports connection:true","v":0,"c0eead19-4586-47fd-9018-7a0ce8481009 WaitActive WaitActive WaitActive"name":,""v"crucible:"0,,""levelname""::"30crucible","level":30,"time,"":"time":"2023-09-22T23:21:37.755791299Z"2023-09-22T23:21:37.755793363Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:":47694769}}

{{"msg":""msg":"c8c6d6f5-1396-4842-801b-a86e7e53a2f6 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible[0] c0eead19-4586-47fd-9018-7a0ce8481009 (64e4856d-2635-4307-a4f9-48bc00b7c13c) WaitActive WaitActive WaitActive ds_transition to WaitQuorum"",","levelv""::300,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.755826811Z,""time,"":"hostname":"2023-09-22T23:21:37.75583027Z","ip-10-150-1-55.us-west-2.compute.internal"hostname",:""pid":4769}
ip-10-150-1-55.us-west-2.compute.internal","pid":4769{}
"msg":"{[0]R  flush_numbers: [0, 0]"","msg"v:"":0,"name":"[0] Transition from WaitActive to WaitQuorum"crucible",,""v"level:"0:,30"name":"crucible","level":30,"time":","2023-09-22T23:21:37.755860746Ztime"":","hostname":2023-09-22T23:21:37.75586376Z"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769ip-10-150-1-55.us-west-2.compute.internal","}
pid":4769}
{{"msg":""msg":"[0]R  generation: [0, 0]","v[0] new RM replaced this: None"":0,","v":name0":,""namecrucible"":","crucible"level",:"30level":40,,""timetime""::""2023-09-22T23:21:37.75589539Z"2023-09-22T23:21:37.755896067Z",","hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid:":47694769}}

{{"msg":""msg":"[0]R  dirty: [false, false]","[0] Starts reconcile loop"v":,0",v"":name0":,""namecrucible"":","level":30crucible",",time"":"level":302023-09-22T23:21:37.755929423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,{"time":""msg":"2023-09-22T23:21:37.755940961Z","[1]R  flush_numbers: [0, 0]"hostname",:""v":0,"name"ip-10-150-1-55.us-west-2.compute.internal:"","cruciblepid"":,4769"level"}:
30{"msg":"[1] c0eead19-4586-47fd-9018-7a0ce8481009 (64e4856d-2635-4307-a4f9-48bc00b7c13c) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum,""time,"":"v":0,"2023-09-22T23:21:37.755963207Zname"":","crucible"hostname",:""level":30ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":,""time":"[1]R  generation: [0, 0]","v2023-09-22T23:21:37.75597915Z"":0,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{,"name"":msg"":"crucible","level":[1] Transition from WaitActive to WaitQuorum"30,"v":0,"name":"crucible","level":30,"time":","time"2023-09-22T23:21:37.756005514Z:"","hostname"2023-09-22T23:21:37.756009979Z:"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal4769","}pid
":4769}
{{"msg":""msg":"[1]R  dirty: [false, false]","[1] new RM replaced this: Nonev"":,0",v"":name0":,""namecrucible"":","crucible"level",:"30level":40,,""timetime""::""2023-09-22T23:21:37.756042808Z2023-09-22T23:21:37.756042109Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid:":47694769}}

{"{msg":""msg":"[1] Starts reconcile loop","[2]R  flush_numbers: [0, 0]v"":,0"v,"":0name",:""name"crucible:"","crucible"level,"":30level":30,,""timetime""::""2023-09-22T23:21:37.75608338Z"2023-09-22T23:21:37.756082822Z",","hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid:":47694769}}
{"msg":"
[2]R  generation: [0, 0]","v":0,"name{":"crucible",""levelmsg""::"30[2] c0eead19-4586-47fd-9018-7a0ce8481009 (64e4856d-2635-4307-a4f9-48bc00b7c13c) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible",",level"":time"30:"2023-09-22T23:21:37.756119501Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"{2023-09-22T23:21:37.756131141Z",""msghostname""::""[2]R  dirty: [false, false]"ip-10-150-1-55.us-west-2.compute.internal",",pid""v:":47690,}"
name":"crucible"{,"level":"30msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.756162154Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal:"","pid":47692023-09-22T23:21:37.756169719Z",}"
hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid"":msg4769":"}
Max found gen is 1","v"{:0,""name"msg:"":"crucible","[2] new RM replaced this: Nonelevel"":,"30v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:37.756197744Z",","time":hostname"":"2023-09-22T23:21:37.75620177Z","ip-10-150-1-55.us-west-2.compute.internal"hostname,"":"pid":4769}
ip-10-150-1-55.us-west-2.compute.internal","pid":4769{}
"msg":"{Generation requested: 1 >= found:1"","msg"v:"":0,"[2] Starts reconcile loopname"":","vcrucible"":0,","level"name:":30"crucible","level":30,"time":","time":"2023-09-22T23:21:37.756231817Z","2023-09-22T23:21:37.756234134Z"hostname":,""hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid"ip-10-150-1-55.us-west-2.compute.internal:"4769,"pid}"
:4769}
{{"msg":""msg"Next flush: 1:"","v":0,"[0] 127.0.0.1:35164 task reports connection:true"name":,""v"crucible:"0,,""levelname""::"30crucible","level":30,"time",:""time":"2023-09-22T23:21:37.756265546Z",2023-09-22T23:21:37.756267603Z""hostname,"":"hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internal"pid",:"4769pid":}4769
}
{"{msg":""All extents matchmsg"":","v":0,"name":"c0eead19-4586-47fd-9018-7a0ce8481009 WaitQuorum WaitQuorum WaitQuorumcrucible"",,""vlevel""::030,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.756298399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
,"time":"{2023-09-22T23:21:37.756301636Z"","msg":hostname"":"No downstairs repair required","ip-10-150-1-55.us-west-2.compute.internal"v",:"0pid",:"4769name":"}
crucible","level":30{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.756332226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769},
"time":"{2023-09-22T23:21:37.756341204Z"","msg":"hostname":"No initial repair work was required",ip-10-150-1-55.us-west-2.compute.internal""v,"":0pid":,"4769name":}"
crucible","level"{:30"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.756366343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid",:"4769time":}"
2023-09-22T23:21:37.756374365Z","hostname{":""msg":"ip-10-150-1-55.us-west-2.compute.internal","pid":Set Downstairs and Upstairs active4769}
{""msg":","v"[0]R  dirty: [false, false]:"0,,""v"name:"0:","cruciblename"":","crucible"level",:"30level":30,,""timetime""::""2023-09-22T23:21:37.756405419Z2023-09-22T23:21:37.756405809Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",","pid"pid:":47694769}}

{"{msg":""msg[1]R  flush_numbers: [0, 0]"":","v":0,"name":"crucible","c8c6d6f5-1396-4842-801b-a86e7e53a2f6 is now active with session: ddd3f126-3dca-46a7-9147-f6bc461d9852level"":,30"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.756440644Z",,""time"hostname:"":"2023-09-22T23:21:37.75644493Z",ip-10-150-1-55.us-west-2.compute.internal"","hostname"pid:"":4769}
ip-10-150-1-55.us-west-2.compute.internal","pid":4769{}
"msg":"{[1]R  generation: [0, 0]",""msgv""::"0,"name":"crucible",c8c6d6f5-1396-4842-801b-a86e7e53a2f6 Set Active after no repair""level,"":v30":0,"name":"crucible","level":30,"time":","2023-09-22T23:21:37.75648007Ztime"":","hostname":"2023-09-22T23:21:37.756483794Z","hostname":ip-10-150-1-55.us-west-2.compute.internal"","pid":4769ip-10-150-1-55.us-west-2.compute.internal"},
"pid":4769{}
"msg":"{[1]R  dirty: [false, false]"",msg"":v"":0,"name":"Notify all downstairs, region set compare is done."crucible",,""v"level:"0:,30"name":"crucible","level":30,"time":","time"2023-09-22T23:21:37.756514233Z:"","hostname"2023-09-22T23:21:37.756517258Z:"","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal4769","}pid
":4769}
{"{msg":""msg":"[2]R  flush_numbers: [0, 0]","Set check for repair"v":,0",v"":name0":,""namecrucible"":","cruciblelevel"":,"30level":30,,""timetime""::""2023-09-22T23:21:37.75654908Z2023-09-22T23:21:37.756549943Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pid"pid:":47694769}}

{{"msg":""msg":"[2]R  generation: [0, 0]","v":0[1] 127.0.0.1:60947 task reports connection:true,"",name"":v"":0crucible",","name"level:"":30crucible","level":30,"time":"2023-09-22T23:21:37.756585049Z","hostname":","time"ip-10-150-1-55.us-west-2.compute.internal:"","pid":47692023-09-22T23:21:37.756583394Z"},"
hostname":"{ip-10-150-1-55.us-west-2.compute.internal","pid"":msg"4769:"}
c8c6d6f5-1396-4842-801b-a86e7e53a2f6 Active Active Active","v":0{,"name":""cruciblemsg"":,""level":30[2]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.756619584Z","hostname":,""time":"ip-10-150-1-55.us-west-2.compute.internal",2023-09-22T23:21:37.756624865Z""pid",:"4769hostname":}"
ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"{msg":"Max found gen is 1""msg,"":"v":0Set check for repair,""name,"":"v":crucible0",","name":level"":crucible30","level":30,"time":","time":"2023-09-22T23:21:37.756656638Z","2023-09-22T23:21:37.756658359Zhostname"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4769,"pid}":
4769}
{"{msg":""msg":"Generation requested: 1 >= found:1","v":0[2] 127.0.0.1:51595 task reports connection:true",",name"":v"":0crucible",","name":level"":crucible30","level":30,","time":time"":"2023-09-22T23:21:37.756690987Z"2023-09-22T23:21:37.756692512Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:":47694769}}

{"{msg":""msgNext flush: 1"":","v":0,"name"c8c6d6f5-1396-4842-801b-a86e7e53a2f6 Active Active Active:"","crucible"v",:"0level,"":30name":"crucible","level":30,"time":","time":2023-09-22T23:21:37.756723387Z"","hostname2023-09-22T23:21:37.756726041Z"":","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":ip-10-150-1-55.us-west-2.compute.internal"4769,"}pid
":4769}
{"{msg":""msgAll extents match"":","v":Set check for repair0",","name"v:"":0crucible,"","name":level"":30crucible","level":30,","time"time:"":"2023-09-22T23:21:37.756756522Z"2023-09-22T23:21:37.756758202Z",","hostname"hostname:"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal",","pid"pid:":47694769}}

{"{msg":""msg":"No downstairs repair required","v"[0] received reconcile message:"0,,""v"name:"0:","cruciblename"":,""cruciblelevel"":,"30level":30,,""timetime""::""2023-09-22T23:21:37.75679059Z2023-09-22T23:21:37.756789914Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47694769}}

{"{msg":""msg":"No initial repair work was required","v[0] All repairs completed, exit"":0,","v"name:"0:","cruciblename"":,""cruciblelevel"":,"30level":30,,""timetime""::""2023-09-22T23:21:37.756829892Z2023-09-22T23:21:37.756829245Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47694769}}
{"msg":"
[0] Starts cmd_loop","v":0,"{name":"crucible"","msg"level"::"30Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.756865757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal",","time"pid:"":4769}2023-09-22T23:21:37.756872428Z
","hostname":{""msg"ip-10-150-1-55.us-west-2.compute.internal:"","pid":4769[1] received reconcile message"},
"v":0,"{name":"crucible""msg,"":"level":30c0eead19-4586-47fd-9018-7a0ce8481009 is now active with session: 64e4856d-2635-4307-a4f9-48bc00b7c13c","v":0,"name":"crucible","level":30,,""time"time:"":"2023-09-22T23:21:37.756908208Z"2023-09-22T23:21:37.756900139Z",","hostname":hostname"":"ip-10-150-1-55.us-west-2.compute.internal"ip-10-150-1-55.us-west-2.compute.internal,"","pid"pid:":47694769}}

{"{msg":""msg":"c0eead19-4586-47fd-9018-7a0ce8481009 Set Active after no repair"[1] All repairs completed, exit",","v"v:"0:0,","name"name:"":"crucible"crucible",","level"level:":3030,,""timetime""::""2023-09-22T23:21:37.756942739Z2023-09-22T23:21:37.756942517Z"",,""hostnamehostname""::""ip-10-150-1-55.us-west-2.compute.internalip-10-150-1-55.us-west-2.compute.internal"",,""pidpid""::47694769}}

{"{msg":""msg"[1] Starts cmd_loop:"","v":0,"Notify all downstairs, region set compare is done.name"":","cruciblev"":,0","level"name:":30"crucible","level":30,"time":","time":"2023-09-22T23:21:37.756977291Z","2023-09-22T23:21:37.756979553Z"hostname",:""hostname":"ip-10-150-1-55.us-west-2.compute.internal","ip-10-150-1-55.us-west-2.compute.internalpid"":,"4769pid":}4769
}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","{level":30"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757008937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"}
time":"{2023-09-22T23:21:37.757017163Z",""hostnamemsg""::""[2] All repairs completed, exit"ip-10-150-1-55.us-west-2.compute.internal",,""v"pid:"0:,4769"name"}:
"crucible","{level":30"msg":"[1] 127.0.0.1:62068 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757043339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"}time
":"{2023-09-22T23:21:37.757052129Z",""hostnamemsg""::""[2] Starts cmd_loop",ip-10-150-1-55.us-west-2.compute.internal""v,"":0pid",:"4769name":"}
crucible","level"{:30"msg":"c0eead19-4586-47fd-9018-7a0ce8481009 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757076218Z","hostname",":time"":"2023-09-22T23:21:37.757085698Z"ip-10-150-1-55.us-west-2.compute.internal",","hostnamepid""::"4769}
ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757118022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 127.0.0.1:42987 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757152318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"c0eead19-4586-47fd-9018-7a0ce8481009 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757182203Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757209112Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757231781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757250904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757274096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757293966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757321013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757341672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757361008Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.75737997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.757398741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.758 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.758 INFO Opened existing region file "/tmp/downstairs-LHGaEZWT/region.json"
Sep 22 23:21:37.758 INFO Database read version 1
Sep 22 23:21:37.758 INFO Database write version 1
test test::integration_test_io_out_of_range ... ok
Sep 22 23:21:37.758 DEBG Read      :1005 deps:[JobId(1004), JobId(1002), JobId(1000)] res:true
Sep 22 23:21:37.759 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.759 INFO Created new region file "/tmp/downstairs-Was11KKc/region.json"
test test::integration_test_guest_zero_length_io ... ok
Sep 22 23:21:37.759 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.759 DEBG Read      :1005 deps:[JobId(1004), JobId(1002), JobId(1000)] res:true
Sep 22 23:21:37.759 INFO Created new region file "/tmp/downstairs-0VvjfZUg/region.json"
test test::integration_test_io_span_out_of_range ... ok
Sep 22 23:21:37.760 INFO UUID: 838f9715-247a-451d-9f6e-d47e6fb7eb32
Sep 22 23:21:37.760 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.760 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.760 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.760 INFO Using address: 127.0.0.1:60919, task: main
Sep 22 23:21:37.760 DEBG Read      :1005 deps:[JobId(1004), JobId(1002), JobId(1000)] res:true
Sep 22 23:21:37.760 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.760 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.760 INFO Created new region file "/tmp/downstairs-iykT2e0l/region.json"
Sep 22 23:21:37.760 INFO Opened existing region file "/tmp/downstairs-P4Ld1XOk/region.json"
Sep 22 23:21:37.760 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.760 INFO Database read version 1
Sep 22 23:21:37.760 INFO Database write version 1
Sep 22 23:21:37.760 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47350, task: repair
Sep 22 23:21:37.760 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.760 INFO Opened existing region file "/tmp/downstairs-6LMZkwp5/region.json"
Sep 22 23:21:37.760 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47350, task: repair
Sep 22 23:21:37.760 INFO Database read version 1
Sep 22 23:21:37.760 INFO Database write version 1
Sep 22 23:21:37.760 INFO listening, local_addr: 127.0.0.1:47350, task: repair
Sep 22 23:21:37.761 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47350, task: repair
Sep 22 23:21:37.761 INFO Using repair address: 127.0.0.1:47350, task: main
Sep 22 23:21:37.761 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.762 INFO Upstairs starts
Sep 22 23:21:37.762 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.762 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.762 INFO Crucible stats registered with UUID: 6afc946f-d080-46eb-8409-d4d69f1e5cf5
Sep 22 23:21:37.762 INFO Crucible 6afc946f-d080-46eb-8409-d4d69f1e5cf5 has session id: c2a95475-2918-450f-b16e-7b810ca0119f
Sep 22 23:21:37.762 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.762 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.762 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.762 INFO [0] connecting to 127.0.0.1:35178, looper: 0
Sep 22 23:21:37.762 INFO [1] connecting to 127.0.0.1:54378, looper: 1
Sep 22 23:21:37.762 INFO UUID: d57f7bca-56d6-4fef-ba3b-7c8c7e1674fe
Sep 22 23:21:37.762 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.762 INFO UUID: c675aa7b-d28f-41ce-bbbd-77c754910d5f
Sep 22 23:21:37.762 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.762 INFO [2] connecting to 127.0.0.1:60919, looper: 2
Sep 22 23:21:37.762 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.762 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.762 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.762 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.762 INFO Using address: 127.0.0.1:46846, task: main
Sep 22 23:21:37.762 INFO Using address: 127.0.0.1:48840, task: main
Sep 22 23:21:37.763 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.763 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.763 INFO Flush timeout: 0.5
Sep 22 23:21:37.763 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.763 INFO accepted connection from 127.0.0.1:43367, task: main
Sep 22 23:21:37.763 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.763 INFO [0] 6afc946f-d080-46eb-8409-d4d69f1e5cf5 looper connected, looper: 0
Sep 22 23:21:37.763 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48561, task: repair
Sep 22 23:21:37.763 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38246, task: repair
Sep 22 23:21:37.763 INFO [0] Proc runs for 127.0.0.1:35178 in state New
Sep 22 23:21:37.763 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48561, task: repair
Sep 22 23:21:37.763 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38246, task: repair
Sep 22 23:21:37.763 INFO listening, local_addr: 127.0.0.1:48561, task: repair
Sep 22 23:21:37.763 INFO listening, local_addr: 127.0.0.1:38246, task: repair
Sep 22 23:21:37.763 INFO [1] 6afc946f-d080-46eb-8409-d4d69f1e5cf5 looper connected, looper: 1
Sep 22 23:21:37.763 INFO [1] Proc runs for 127.0.0.1:54378 in state New
Sep 22 23:21:37.763 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.763 INFO [2] 6afc946f-d080-46eb-8409-d4d69f1e5cf5 looper connected, looper: 2
Sep 22 23:21:37.763 INFO Opened existing region file "/tmp/downstairs-0VvjfZUg/region.json"
Sep 22 23:21:37.763 INFO Database read version 1
Sep 22 23:21:37.763 INFO Database write version 1
Sep 22 23:21:37.763 INFO [2] Proc runs for 127.0.0.1:60919 in state New
Sep 22 23:21:37.763 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48561, task: repair
Sep 22 23:21:37.763 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38246, task: repair
Sep 22 23:21:37.763 INFO Using repair address: 127.0.0.1:38246, task: main
Sep 22 23:21:37.763 INFO Using repair address: 127.0.0.1:48561, task: main
Sep 22 23:21:37.763 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.763 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.763 INFO accepted connection from 127.0.0.1:58083, task: main
Sep 22 23:21:37.763 INFO accepted connection from 127.0.0.1:39236, task: main
Sep 22 23:21:37.764 INFO Connection request from 6afc946f-d080-46eb-8409-d4d69f1e5cf5 with version 4, task: proc
Sep 22 23:21:37.764 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.764 INFO upstairs UpstairsConnection { upstairs_id: 6afc946f-d080-46eb-8409-d4d69f1e5cf5, session_id: a5cb2416-c55e-442a-875e-b2f1c82b7925, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.764 INFO Connection request from 6afc946f-d080-46eb-8409-d4d69f1e5cf5 with version 4, task: proc
Sep 22 23:21:37.764 INFO Created new region file "/tmp/downstairs-kyT9vuNL/region.json"
Sep 22 23:21:37.764 INFO Upstairs starts
Sep 22 23:21:37.764 INFO upstairs UpstairsConnection { upstairs_id: 6afc946f-d080-46eb-8409-d4d69f1e5cf5, session_id: a5cb2416-c55e-442a-875e-b2f1c82b7925, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.764 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.764 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.764 INFO Crucible stats registered with UUID: be1bbe57-a78b-41fd-91a6-742ef82c740e
Sep 22 23:21:37.764 INFO Connection request from 6afc946f-d080-46eb-8409-d4d69f1e5cf5 with version 4, task: proc
Sep 22 23:21:37.764 INFO Crucible be1bbe57-a78b-41fd-91a6-742ef82c740e has session id: 777b8bd2-886b-4a1b-bd27-823aabe06e49
Sep 22 23:21:37.764 INFO upstairs UpstairsConnection { upstairs_id: 6afc946f-d080-46eb-8409-d4d69f1e5cf5, session_id: a5cb2416-c55e-442a-875e-b2f1c82b7925, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.764 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.764 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.764 INFO Opened existing region file "/tmp/downstairs-iykT2e0l/region.json"
Sep 22 23:21:37.764 INFO Database read version 1
Sep 22 23:21:37.764 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.764 INFO Database write version 1
Sep 22 23:21:37.764 INFO [0] 6afc946f-d080-46eb-8409-d4d69f1e5cf5 (a5cb2416-c55e-442a-875e-b2f1c82b7925) New New New ds_transition to WaitActive
Sep 22 23:21:37.764 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.764 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.764 INFO [0] connecting to 127.0.0.1:61806, looper: 0
Sep 22 23:21:37.764 INFO UUID: 948c8c95-e366-4ab4-b43c-f85926d90cc9
Sep 22 23:21:37.764 INFO [1] 6afc946f-d080-46eb-8409-d4d69f1e5cf5 (a5cb2416-c55e-442a-875e-b2f1c82b7925) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.764 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.764 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.764 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.764 INFO [2] 6afc946f-d080-46eb-8409-d4d69f1e5cf5 (a5cb2416-c55e-442a-875e-b2f1c82b7925) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.764 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.764 INFO Using address: 127.0.0.1:63732, task: main
Sep 22 23:21:37.764 INFO [1] connecting to 127.0.0.1:36272, looper: 1
Sep 22 23:21:37.764 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.765 INFO [2] connecting to 127.0.0.1:46846, looper: 2
The guest has requested activation
Sep 22 23:21:37.765 INFO 6afc946f-d080-46eb-8409-d4d69f1e5cf5 active request set
Sep 22 23:21:37.765 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.765 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.765 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.765 INFO Flush timeout: 0.5
Sep 22 23:21:37.765 INFO [0] received activate with gen 1
Sep 22 23:21:37.765 INFO [0] client got ds_active_rx, promote! session a5cb2416-c55e-442a-875e-b2f1c82b7925
Sep 22 23:21:37.765 INFO [1] received activate with gen 1
Sep 22 23:21:37.765 INFO [1] client got ds_active_rx, promote! session a5cb2416-c55e-442a-875e-b2f1c82b7925
Sep 22 23:21:37.765 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52997, task: repair
Sep 22 23:21:37.765 INFO [2] received activate with gen 1
Sep 22 23:21:37.765 INFO [2] client got ds_active_rx, promote! session a5cb2416-c55e-442a-875e-b2f1c82b7925
Sep 22 23:21:37.765 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52997, task: repair
Sep 22 23:21:37.765 INFO listening, local_addr: 127.0.0.1:52997, task: repair
Sep 22 23:21:37.765 INFO accepted connection from 127.0.0.1:50466, task: main
Sep 22 23:21:37.765 INFO accepted connection from 127.0.0.1:41992, task: main
Sep 22 23:21:37.765 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52997, task: repair
Sep 22 23:21:37.765 INFO accepted connection from 127.0.0.1:59307, task: main
Sep 22 23:21:37.765 INFO Using repair address: 127.0.0.1:52997, task: main
Sep 22 23:21:37.765 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.765 INFO [0] be1bbe57-a78b-41fd-91a6-742ef82c740e looper connected, looper: 0
Sep 22 23:21:37.765 INFO [0] Proc runs for 127.0.0.1:61806 in state New
Sep 22 23:21:37.765 INFO UUID: a6815f0f-5f43-4904-a056-e79c5267f195
Sep 22 23:21:37.765 INFO [1] be1bbe57-a78b-41fd-91a6-742ef82c740e looper connected, looper: 1
Sep 22 23:21:37.765 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.765 INFO [1] Proc runs for 127.0.0.1:36272 in state New
Sep 22 23:21:37.765 INFO [0] downstairs client at 127.0.0.1:35178 has UUID a32c7e82-e087-4524-b977-a5b003758c5f
Sep 22 23:21:37.765 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a32c7e82-e087-4524-b977-a5b003758c5f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.765 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.765 INFO [2] be1bbe57-a78b-41fd-91a6-742ef82c740e looper connected, looper: 2
Sep 22 23:21:37.765 INFO 6afc946f-d080-46eb-8409-d4d69f1e5cf5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.765 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.766 INFO Using address: 127.0.0.1:60322, task: main
Sep 22 23:21:37.766 INFO [2] Proc runs for 127.0.0.1:46846 in state New
Sep 22 23:21:37.766 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.766 INFO [1] downstairs client at 127.0.0.1:54378 has UUID 6c77c5f6-1f7d-446d-9f57-ac7866e05802
Sep 22 23:21:37.766 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6c77c5f6-1f7d-446d-9f57-ac7866e05802, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.766 INFO 6afc946f-d080-46eb-8409-d4d69f1e5cf5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.766 INFO Created new region file "/tmp/downstairs-2YrrmjVu/region.json"
Sep 22 23:21:37.766 INFO [2] downstairs client at 127.0.0.1:60919 has UUID 838f9715-247a-451d-9f6e-d47e6fb7eb32
Sep 22 23:21:37.766 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 838f9715-247a-451d-9f6e-d47e6fb7eb32, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.766 INFO 6afc946f-d080-46eb-8409-d4d69f1e5cf5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.766 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.766 INFO Connection request from be1bbe57-a78b-41fd-91a6-742ef82c740e with version 4, task: proc
Sep 22 23:21:37.766 INFO upstairs UpstairsConnection { upstairs_id: be1bbe57-a78b-41fd-91a6-742ef82c740e, session_id: dfde9f11-1418-46ce-85ab-4ed932ebdb6b, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.766 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.766 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46041, task: repair
Sep 22 23:21:37.766 INFO Connection request from be1bbe57-a78b-41fd-91a6-742ef82c740e with version 4, task: proc
Sep 22 23:21:37.766 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46041, task: repair
Sep 22 23:21:37.766 INFO upstairs UpstairsConnection { upstairs_id: be1bbe57-a78b-41fd-91a6-742ef82c740e, session_id: dfde9f11-1418-46ce-85ab-4ed932ebdb6b, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.766 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.766 INFO listening, local_addr: 127.0.0.1:46041, task: repair
Sep 22 23:21:37.766 INFO Connection request from be1bbe57-a78b-41fd-91a6-742ef82c740e with version 4, task: proc
Sep 22 23:21:37.766 INFO upstairs UpstairsConnection { upstairs_id: be1bbe57-a78b-41fd-91a6-742ef82c740e, session_id: dfde9f11-1418-46ce-85ab-4ed932ebdb6b, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.766 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.766 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46041, task: repair
Sep 22 23:21:37.766 INFO Using repair address: 127.0.0.1:46041, task: main
Sep 22 23:21:37.766 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.766 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.766 INFO [0] be1bbe57-a78b-41fd-91a6-742ef82c740e (dfde9f11-1418-46ce-85ab-4ed932ebdb6b) New New New ds_transition to WaitActive
Sep 22 23:21:37.766 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.766 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.766 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.767 INFO [1] be1bbe57-a78b-41fd-91a6-742ef82c740e (dfde9f11-1418-46ce-85ab-4ed932ebdb6b) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.767 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.767 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.767 INFO Created new region file "/tmp/downstairs-95N00kmj/region.json"
Sep 22 23:21:37.767 INFO [2] be1bbe57-a78b-41fd-91a6-742ef82c740e (dfde9f11-1418-46ce-85ab-4ed932ebdb6b) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.767 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.767 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.767 INFO Opened existing region file "/tmp/downstairs-kyT9vuNL/region.json"
Sep 22 23:21:37.767 INFO Database read version 1
Sep 22 23:21:37.767 INFO [0] 6afc946f-d080-46eb-8409-d4d69f1e5cf5 (a5cb2416-c55e-442a-875e-b2f1c82b7925) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.767 INFO Database write version 1
Sep 22 23:21:37.767 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.767 WARN [0] new RM replaced this: None
Sep 22 23:21:37.767 INFO [0] Starts reconcile loop
The guest has requested activation
Sep 22 23:21:37.767 INFO [1] 6afc946f-d080-46eb-8409-d4d69f1e5cf5 (a5cb2416-c55e-442a-875e-b2f1c82b7925) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.767 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.767 WARN [1] new RM replaced this: None
Sep 22 23:21:37.767 INFO [1] Starts reconcile loop
Sep 22 23:21:37.767 INFO be1bbe57-a78b-41fd-91a6-742ef82c740e active request set
Sep 22 23:21:37.767 INFO [2] 6afc946f-d080-46eb-8409-d4d69f1e5cf5 (a5cb2416-c55e-442a-875e-b2f1c82b7925) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.767 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.767 WARN [2] new RM replaced this: None
Sep 22 23:21:37.767 INFO [2] Starts reconcile loop
Sep 22 23:21:37.767 INFO [0] received activate with gen 1
Sep 22 23:21:37.767 INFO [0] client got ds_active_rx, promote! session dfde9f11-1418-46ce-85ab-4ed932ebdb6b
Sep 22 23:21:37.767 INFO [0] 127.0.0.1:35178 task reports connection:true
Sep 22 23:21:37.767 INFO 6afc946f-d080-46eb-8409-d4d69f1e5cf5 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.767 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.767 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.767 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.767 INFO [1] received activate with gen 1
Sep 22 23:21:37.767 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.767 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.767 INFO [1] client got ds_active_rx, promote! session dfde9f11-1418-46ce-85ab-4ed932ebdb6b
Sep 22 23:21:37.767 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.767 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.767 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.767 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.767 INFO Max found gen is 1
Sep 22 23:21:37.767 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.767 INFO Next flush: 1
Sep 22 23:21:37.767 INFO [2] received activate with gen 1
Sep 22 23:21:37.767 INFO All extents match
Sep 22 23:21:37.767 INFO No downstairs repair required
Sep 22 23:21:37.767 INFO [2] client got ds_active_rx, promote! session dfde9f11-1418-46ce-85ab-4ed932ebdb6b
Sep 22 23:21:37.767 INFO No initial repair work was required
Sep 22 23:21:37.767 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.767 INFO 6afc946f-d080-46eb-8409-d4d69f1e5cf5 is now active with session: a5cb2416-c55e-442a-875e-b2f1c82b7925
Sep 22 23:21:37.767 INFO 6afc946f-d080-46eb-8409-d4d69f1e5cf5 Set Active after no repair
Sep 22 23:21:37.767 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.767 INFO Set check for repair
Sep 22 23:21:37.767 INFO [1] 127.0.0.1:54378 task reports connection:true
Sep 22 23:21:37.767 INFO 6afc946f-d080-46eb-8409-d4d69f1e5cf5 Active Active Active
Sep 22 23:21:37.767 INFO Set check for repair
Sep 22 23:21:37.767 INFO [2] 127.0.0.1:60919 task reports connection:true
Sep 22 23:21:37.767 INFO 6afc946f-d080-46eb-8409-d4d69f1e5cf5 Active Active Active
Sep 22 23:21:37.767 INFO Set check for repair
Sep 22 23:21:37.767 INFO [0] received reconcile message
Sep 22 23:21:37.767 INFO [0] All repairs completed, exit
Sep 22 23:21:37.767 INFO [0] Starts cmd_loop
Sep 22 23:21:37.768 INFO [1] received reconcile message
Sep 22 23:21:37.768 INFO [1] All repairs completed, exit
Sep 22 23:21:37.768 INFO [1] Starts cmd_loop
Sep 22 23:21:37.768 INFO [2] received reconcile message
Sep 22 23:21:37.768 INFO [2] All repairs completed, exit
Sep 22 23:21:37.768 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:37.768 INFO [0] downstairs client at 127.0.0.1:61806 has UUID 17366ba3-9d52-4dce-a4fd-d4657621aec9
Sep 22 23:21:37.768 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 17366ba3-9d52-4dce-a4fd-d4657621aec9, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.768 INFO be1bbe57-a78b-41fd-91a6-742ef82c740e WaitActive WaitActive WaitActive
Sep 22 23:21:37.768 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.768 INFO [1] downstairs client at 127.0.0.1:36272 has UUID e173e972-e8cf-416e-bc11-1d8d35f2e9ba
Sep 22 23:21:37.768 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e173e972-e8cf-416e-bc11-1d8d35f2e9ba, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.768 INFO be1bbe57-a78b-41fd-91a6-742ef82c740e WaitActive WaitActive WaitActive
Sep 22 23:21:37.768 INFO [2] downstairs client at 127.0.0.1:46846 has UUID d57f7bca-56d6-4fef-ba3b-7c8c7e1674fe
Sep 22 23:21:37.768 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d57f7bca-56d6-4fef-ba3b-7c8c7e1674fe, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.768 INFO be1bbe57-a78b-41fd-91a6-742ef82c740e WaitActive WaitActive WaitActive
Sep 22 23:21:37.768 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.768 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.769 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.769 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.769 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.769 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.769 INFO UUID: 59952ac1-1e6e-4726-b89d-5cdada889db9
Sep 22 23:21:37.769 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.769 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.769 INFO Opened existing region file "/tmp/downstairs-2YrrmjVu/region.json"
Sep 22 23:21:37.769 INFO Database read version 1
Sep 22 23:21:37.769 INFO Database write version 1
Sep 22 23:21:37.769 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.769 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.769 INFO Using address: 127.0.0.1:58780, task: main
Sep 22 23:21:37.769 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.769 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.769 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.769 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.770 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.770 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.770 DEBG up_ds_listen was notified
Sep 22 23:21:37.770 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.770 DEBG up_ds_listen process 1000
Sep 22 23:21:37.770 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33158, task: repair
Sep 22 23:21:37.770 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.770 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33158, task: repair
Sep 22 23:21:37.770 INFO listening, local_addr: 127.0.0.1:33158, task: repair
Sep 22 23:21:37.770 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.770 INFO [0] be1bbe57-a78b-41fd-91a6-742ef82c740e (dfde9f11-1418-46ce-85ab-4ed932ebdb6b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.770 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.770 WARN [0] new RM replaced this: None
Sep 22 23:21:37.770 INFO [0] Starts reconcile loop
Sep 22 23:21:37.770 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33158, task: repair
Sep 22 23:21:37.770 INFO Using repair address: 127.0.0.1:33158, task: main
Sep 22 23:21:37.770 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.770 INFO [1] be1bbe57-a78b-41fd-91a6-742ef82c740e (dfde9f11-1418-46ce-85ab-4ed932ebdb6b) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.770 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.770 WARN [1] new RM replaced this: None
Sep 22 23:21:37.770 INFO [1] Starts reconcile loop
Sep 22 23:21:37.770 INFO [2] be1bbe57-a78b-41fd-91a6-742ef82c740e (dfde9f11-1418-46ce-85ab-4ed932ebdb6b) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.770 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.770 WARN [2] new RM replaced this: None
Sep 22 23:21:37.770 INFO [2] Starts reconcile loop
Sep 22 23:21:37.770 INFO [0] 127.0.0.1:61806 task reports connection:true
Sep 22 23:21:37.770 INFO be1bbe57-a78b-41fd-91a6-742ef82c740e WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.770 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.770 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.770 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.770 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.770 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.770 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.770 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.770 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.770 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.770 INFO Max found gen is 1
Sep 22 23:21:37.770 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.771 INFO Next flush: 1
Sep 22 23:21:37.771 INFO All extents match
Sep 22 23:21:37.771 INFO No downstairs repair required
Sep 22 23:21:37.771 INFO No initial repair work was required
Sep 22 23:21:37.771 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.771 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.771 INFO be1bbe57-a78b-41fd-91a6-742ef82c740e is now active with session: dfde9f11-1418-46ce-85ab-4ed932ebdb6b
Sep 22 23:21:37.771 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.771 INFO be1bbe57-a78b-41fd-91a6-742ef82c740e Set Active after no repair
Sep 22 23:21:37.771 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.771 WARN 7743f09f-f465-4fba-892d-457fc58d9abf request to replace downstairs 127.0.0.1:45992 with 127.0.0.1:48840
Sep 22 23:21:37.771 INFO Set check for repair
Sep 22 23:21:37.771 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.771 INFO 7743f09f-f465-4fba-892d-457fc58d9abf found old target: 127.0.0.1:45992 at 0
Sep 22 23:21:37.771 INFO Opened existing region file "/tmp/downstairs-95N00kmj/region.json"
Sep 22 23:21:37.771 INFO Database read version 1
Sep 22 23:21:37.771 INFO [1] 127.0.0.1:36272 task reports connection:true
Sep 22 23:21:37.771 INFO 7743f09f-f465-4fba-892d-457fc58d9abf replacing old: 127.0.0.1:45992 at 0
Sep 22 23:21:37.771 INFO Database write version 1
Sep 22 23:21:37.771 INFO be1bbe57-a78b-41fd-91a6-742ef82c740e Active Active Active
Sep 22 23:21:37.771 INFO [0] client skip 0 in process jobs because fault, : downstairs
Sep 22 23:21:37.771 INFO Set check for repair
Sep 22 23:21:37.771 INFO [0] changed 0 jobs to fault skipped, : downstairs
Sep 22 23:21:37.771 INFO [0] 7743f09f-f465-4fba-892d-457fc58d9abf (3175f1fa-0abb-4faa-9a83-0df20993aaa9) Active Active Active ds_transition to Replacing
Sep 22 23:21:37.771 INFO [0] Transition from Active to Replacing
Sep 22 23:21:37.771 INFO [2] 127.0.0.1:46846 task reports connection:true
Sep 22 23:21:37.771 INFO be1bbe57-a78b-41fd-91a6-742ef82c740e Active Active Active
Sep 22 23:21:37.771 INFO Set check for repair
Sep 22 23:21:37.771 INFO [0] received reconcile message
Sep 22 23:21:37.771 WARN 7743f09f-f465-4fba-892d-457fc58d9abf request to replace downstairs 127.0.0.1:57990 with 127.0.0.1:58780
Sep 22 23:21:37.771 INFO [0] All repairs completed, exit
Sep 22 23:21:37.771 INFO 7743f09f-f465-4fba-892d-457fc58d9abf found old target: 127.0.0.1:57990 at 1
Sep 22 23:21:37.771 INFO [0] Starts cmd_loop
Sep 22 23:21:37.771 INFO UUID: d6f117a0-d707-4b2e-a491-e44ab08fd29f
Sep 22 23:21:37.771 INFO [1] received reconcile message
Sep 22 23:21:37.771 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.771 INFO [1] All repairs completed, exit
Sep 22 23:21:37.771 INFO [1] Starts cmd_loop
Sep 22 23:21:37.771 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.771 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.771 INFO Using address: 127.0.0.1:61215, task: main
Sep 22 23:21:37.771 INFO [2] received reconcile message
Sep 22 23:21:37.771 INFO [2] All repairs completed, exit
Sep 22 23:21:37.771 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:37.771 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.772 INFO Upstairs starts
Sep 22 23:21:37.772 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42316, task: repair
Sep 22 23:21:37.772 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42316, task: repair
Sep 22 23:21:37.772 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.772 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.772 INFO listening, local_addr: 127.0.0.1:42316, task: repair
Sep 22 23:21:37.772 INFO Crucible stats registered with UUID: afdd1779-4d01-4718-83ef-1f0582b0fcbd
Sep 22 23:21:37.772 INFO Crucible afdd1779-4d01-4718-83ef-1f0582b0fcbd has session id: d69380f8-391d-43d8-ba26-076f98fc936f
Sep 22 23:21:37.772 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42316, task: repair
Sep 22 23:21:37.772 INFO Using repair address: 127.0.0.1:42316, task: main
Sep 22 23:21:37.772 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.772 INFO [0] connecting to 127.0.0.1:61806, looper: 0
Sep 22 23:21:37.773 INFO [1] connecting to 127.0.0.1:36272, looper: 1
Sep 22 23:21:37.773 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.773 INFO [2] connecting to 127.0.0.1:46846, looper: 2
Sep 22 23:21:37.773 INFO Created new region file "/tmp/downstairs-GSLa5ha2/region.json"
Sep 22 23:21:37.773 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.773 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.773 INFO Flush timeout: 0.5
Sep 22 23:21:37.773 INFO [2] afdd1779-4d01-4718-83ef-1f0582b0fcbd looper connected, looper: 2
Sep 22 23:21:37.773 INFO [2] Proc runs for 127.0.0.1:46846 in state New
Sep 22 23:21:37.773 INFO [0] afdd1779-4d01-4718-83ef-1f0582b0fcbd looper connected, looper: 0
Sep 22 23:21:37.773 INFO [0] Proc runs for 127.0.0.1:61806 in state New
Sep 22 23:21:37.773 INFO [1] afdd1779-4d01-4718-83ef-1f0582b0fcbd looper connected, looper: 1
Sep 22 23:21:37.773 INFO [1] Proc runs for 127.0.0.1:36272 in state New
Sep 22 23:21:37.773 INFO accepted connection from 127.0.0.1:44545, task: main
Sep 22 23:21:37.773 INFO accepted connection from 127.0.0.1:60808, task: main
Sep 22 23:21:37.774 INFO accepted connection from 127.0.0.1:33455, task: main
Sep 22 23:21:37.774 INFO Connection request from afdd1779-4d01-4718-83ef-1f0582b0fcbd with version 4, task: proc
Sep 22 23:21:37.774 INFO upstairs UpstairsConnection { upstairs_id: afdd1779-4d01-4718-83ef-1f0582b0fcbd, session_id: 9c74e4ab-e653-4eaa-b48f-6000a2bdbb12, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.774 INFO UUID: 9d3f8a28-aea4-4f16-a02f-499f4731649b
Sep 22 23:21:37.774 INFO Connection request from afdd1779-4d01-4718-83ef-1f0582b0fcbd with version 4, task: proc
Sep 22 23:21:37.774 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.774 INFO upstairs UpstairsConnection { upstairs_id: afdd1779-4d01-4718-83ef-1f0582b0fcbd, session_id: 9c74e4ab-e653-4eaa-b48f-6000a2bdbb12, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.774 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.774 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.774 INFO Connection request from afdd1779-4d01-4718-83ef-1f0582b0fcbd with version 4, task: proc
Sep 22 23:21:37.774 INFO Using address: 127.0.0.1:48145, task: main
test test::integration_test_just_read ... Sep 22 23:21:37.774 INFO upstairs UpstairsConnection { upstairs_id: afdd1779-4d01-4718-83ef-1f0582b0fcbd, session_id: 9c74e4ab-e653-4eaa-b48f-6000a2bdbb12, gen: 1 } connected, version 4, task: proc
ok
Sep 22 23:21:37.775 INFO [2] afdd1779-4d01-4718-83ef-1f0582b0fcbd (9c74e4ab-e653-4eaa-b48f-6000a2bdbb12) New New New ds_transition to WaitActive
Sep 22 23:21:37.775 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.775 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.775 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42170, task: repair
Sep 22 23:21:37.775 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42170, task: repair
Sep 22 23:21:37.775 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.775 INFO [0] afdd1779-4d01-4718-83ef-1f0582b0fcbd (9c74e4ab-e653-4eaa-b48f-6000a2bdbb12) New New WaitActive ds_transition to WaitActive
Sep 22 23:21:37.775 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.775 INFO listening, local_addr: 127.0.0.1:42170, task: repair
Sep 22 23:21:37.775 INFO [1] afdd1779-4d01-4718-83ef-1f0582b0fcbd (9c74e4ab-e653-4eaa-b48f-6000a2bdbb12) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:21:37.775 INFO Created new region file "/tmp/downstairs-3Ng9gZLV/region.json"
Sep 22 23:21:37.775 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.775 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42170, task: repair
Sep 22 23:21:37.775 INFO Using repair address: 127.0.0.1:42170, task: main
Sep 22 23:21:37.775 INFO No SSL acceptor configured, task: main
The guest has requested activation
Sep 22 23:21:37.775 INFO afdd1779-4d01-4718-83ef-1f0582b0fcbd active request set
Sep 22 23:21:37.775 INFO [0] received activate with gen 1
Sep 22 23:21:37.775 INFO [0] client got ds_active_rx, promote! session 9c74e4ab-e653-4eaa-b48f-6000a2bdbb12
Sep 22 23:21:37.775 INFO [1] received activate with gen 1
Sep 22 23:21:37.775 INFO [1] client got ds_active_rx, promote! session 9c74e4ab-e653-4eaa-b48f-6000a2bdbb12
Sep 22 23:21:37.775 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.775 INFO [2] received activate with gen 1
Sep 22 23:21:37.775 INFO [2] client got ds_active_rx, promote! session 9c74e4ab-e653-4eaa-b48f-6000a2bdbb12
Sep 22 23:21:37.775 INFO Created new region file "/tmp/downstairs-11dCDdeO/region.json"
Sep 22 23:21:37.776 INFO [2] downstairs client at 127.0.0.1:46846 has UUID d57f7bca-56d6-4fef-ba3b-7c8c7e1674fe
Sep 22 23:21:37.776 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d57f7bca-56d6-4fef-ba3b-7c8c7e1674fe, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.776 INFO afdd1779-4d01-4718-83ef-1f0582b0fcbd WaitActive WaitActive WaitActive
Sep 22 23:21:37.776 INFO [0] downstairs client at 127.0.0.1:61806 has UUID 17366ba3-9d52-4dce-a4fd-d4657621aec9
Sep 22 23:21:37.776 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 17366ba3-9d52-4dce-a4fd-d4657621aec9, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.776 INFO afdd1779-4d01-4718-83ef-1f0582b0fcbd WaitActive WaitActive WaitActive
Sep 22 23:21:37.776 INFO [1] downstairs client at 127.0.0.1:36272 has UUID e173e972-e8cf-416e-bc11-1d8d35f2e9ba
Sep 22 23:21:37.776 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e173e972-e8cf-416e-bc11-1d8d35f2e9ba, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.777 INFO afdd1779-4d01-4718-83ef-1f0582b0fcbd WaitActive WaitActive WaitActive
Sep 22 23:21:37.777 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.777 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.777 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.777 INFO Opened existing region file "/tmp/downstairs-GSLa5ha2/region.json"
Sep 22 23:21:37.777 INFO Database read version 1
Sep 22 23:21:37.777 INFO Database write version 1
test test::integration_test_guest_replace_many_downstairs ... ok
Sep 22 23:21:37.778 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.778 INFO Created new region file "/tmp/downstairs-1gO30onD/region.json"
Sep 22 23:21:37.778 INFO Current flush_numbers [0..12]: [0, 0]
test test::integration_test_guest_downstairs_unwritten ... ok
Sep 22 23:21:37.778 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.779 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.779 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.779 INFO Created new region file "/tmp/downstairs-4bgragna/region.json"
Sep 22 23:21:37.779 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.779 INFO [2] afdd1779-4d01-4718-83ef-1f0582b0fcbd (9c74e4ab-e653-4eaa-b48f-6000a2bdbb12) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.779 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.779 WARN [2] new RM replaced this: None
Sep 22 23:21:37.779 INFO [2] Starts reconcile loop
Sep 22 23:21:37.779 INFO [0] afdd1779-4d01-4718-83ef-1f0582b0fcbd (9c74e4ab-e653-4eaa-b48f-6000a2bdbb12) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.779 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.779 WARN [0] new RM replaced this: None
Sep 22 23:21:37.779 INFO [0] Starts reconcile loop
Sep 22 23:21:37.779 INFO [1] afdd1779-4d01-4718-83ef-1f0582b0fcbd (9c74e4ab-e653-4eaa-b48f-6000a2bdbb12) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.779 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.779 WARN [1] new RM replaced this: None
Sep 22 23:21:37.779 INFO [1] Starts reconcile loop
Sep 22 23:21:37.779 INFO [2] 127.0.0.1:46846 task reports connection:true
Sep 22 23:21:37.779 INFO afdd1779-4d01-4718-83ef-1f0582b0fcbd WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.779 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.779 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.779 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.779 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.779 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.779 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.779 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.779 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.779 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.780 INFO Max found gen is 1
Sep 22 23:21:37.780 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.780 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.780 INFO Next flush: 1
Sep 22 23:21:37.780 INFO Opened existing region file "/tmp/downstairs-11dCDdeO/region.json"
Sep 22 23:21:37.780 INFO All extents match
Sep 22 23:21:37.780 INFO Database read version 1
Sep 22 23:21:37.780 INFO No downstairs repair required
Sep 22 23:21:37.780 INFO Database write version 1
Sep 22 23:21:37.780 INFO No initial repair work was required
Sep 22 23:21:37.780 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.780 INFO afdd1779-4d01-4718-83ef-1f0582b0fcbd is now active with session: 9c74e4ab-e653-4eaa-b48f-6000a2bdbb12
Sep 22 23:21:37.780 INFO afdd1779-4d01-4718-83ef-1f0582b0fcbd Set Active after no repair
Sep 22 23:21:37.780 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.780 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.780 INFO Set check for repair
Sep 22 23:21:37.780 INFO UUID: 3ab28074-4b58-4686-8a29-dc39469e56d5
Sep 22 23:21:37.780 INFO Opened existing region file "/tmp/downstairs-3Ng9gZLV/region.json"
Sep 22 23:21:37.780 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.780 INFO Database read version 1
Sep 22 23:21:37.780 INFO Database write version 1
Sep 22 23:21:37.780 INFO [0] 127.0.0.1:61806 task reports connection:true
Sep 22 23:21:37.780 INFO afdd1779-4d01-4718-83ef-1f0582b0fcbd Active Active Active
Sep 22 23:21:37.780 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.780 INFO Set check for repair
Sep 22 23:21:37.780 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.780 INFO Using address: 127.0.0.1:62024, task: main
Sep 22 23:21:37.780 INFO [1] 127.0.0.1:36272 task reports connection:true
Sep 22 23:21:37.780 INFO afdd1779-4d01-4718-83ef-1f0582b0fcbd Active Active Active
Sep 22 23:21:37.780 INFO Set check for repair
Sep 22 23:21:37.780 INFO [0] received reconcile message
Sep 22 23:21:37.780 INFO [0] All repairs completed, exit
Sep 22 23:21:37.780 INFO [0] Starts cmd_loop
Sep 22 23:21:37.780 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.780 INFO [1] received reconcile message
Sep 22 23:21:37.780 INFO [1] All repairs completed, exit
Sep 22 23:21:37.780 INFO [1] Starts cmd_loop
Sep 22 23:21:37.780 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48900, task: repair
Sep 22 23:21:37.780 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48900, task: repair
Sep 22 23:21:37.780 INFO [2] received reconcile message
Sep 22 23:21:37.780 INFO listening, local_addr: 127.0.0.1:48900, task: repair
Sep 22 23:21:37.780 INFO [2] All repairs completed, exit
Sep 22 23:21:37.780 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:37.780 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48900, task: repair
Sep 22 23:21:37.780 INFO Using repair address: 127.0.0.1:48900, task: main
Sep 22 23:21:37.780 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.781 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.781 INFO UUID: 890de4ee-4d21-427c-8570-1ce5a5506828
Sep 22 23:21:37.781 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.781 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.781 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.781 INFO Using address: 127.0.0.1:40738, task: main
Sep 22 23:21:37.781 INFO Upstairs starts
Sep 22 23:21:37.781 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.781 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.781 INFO Crucible stats registered with UUID: c3d60856-e346-48b1-a11b-e1e0e826a086
Sep 22 23:21:37.781 INFO Crucible c3d60856-e346-48b1-a11b-e1e0e826a086 has session id: a8384867-0cbd-477d-8ed0-64521563faca
Sep 22 23:21:37.781 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.781 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.781 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.781 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35328, task: repair
Sep 22 23:21:37.781 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.781 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35328, task: repair
Sep 22 23:21:37.782 INFO [0] connecting to 127.0.0.1:63732, looper: 0
Sep 22 23:21:37.782 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.782 INFO listening, local_addr: 127.0.0.1:35328, task: repair
Sep 22 23:21:37.782 INFO UUID: 8f0829a0-9286-4151-a390-4aadd7c97e80
Sep 22 23:21:37.782 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.782 INFO [1] connecting to 127.0.0.1:61215, looper: 1
Sep 22 23:21:37.782 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.782 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.782 INFO [2] connecting to 127.0.0.1:62024, looper: 2
Sep 22 23:21:37.782 INFO Using address: 127.0.0.1:49534, task: main
Sep 22 23:21:37.782 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35328, task: repair
Sep 22 23:21:37.782 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.782 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.782 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.782 INFO Using repair address: 127.0.0.1:35328, task: main
Sep 22 23:21:37.782 INFO Flush timeout: 0.5
Sep 22 23:21:37.782 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.782 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.782 INFO Opened existing region file "/tmp/downstairs-1gO30onD/region.json"
Sep 22 23:21:37.782 INFO Database read version 1
Sep 22 23:21:37.782 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.782 INFO Database write version 1
Sep 22 23:21:37.782 INFO accepted connection from 127.0.0.1:36783, task: main
Sep 22 23:21:37.782 INFO [0] c3d60856-e346-48b1-a11b-e1e0e826a086 looper connected, looper: 0
Sep 22 23:21:37.782 INFO [0] Proc runs for 127.0.0.1:63732 in state New
Sep 22 23:21:37.782 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.782 INFO accepted connection from 127.0.0.1:39601, task: main
Sep 22 23:21:37.782 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36538, task: repair
Sep 22 23:21:37.782 INFO accepted connection from 127.0.0.1:42280, task: main
Sep 22 23:21:37.782 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36538, task: repair
Sep 22 23:21:37.782 INFO [1] c3d60856-e346-48b1-a11b-e1e0e826a086 looper connected, looper: 1
Sep 22 23:21:37.782 INFO listening, local_addr: 127.0.0.1:36538, task: repair
Sep 22 23:21:37.782 INFO [1] Proc runs for 127.0.0.1:61215 in state New
Sep 22 23:21:37.782 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.782 INFO [2] c3d60856-e346-48b1-a11b-e1e0e826a086 looper connected, looper: 2
Sep 22 23:21:37.782 INFO [2] Proc runs for 127.0.0.1:62024 in state New
Sep 22 23:21:37.782 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.782 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36538, task: repair
Sep 22 23:21:37.782 INFO Using repair address: 127.0.0.1:36538, task: main
Sep 22 23:21:37.783 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.783 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.783 DEBG up_ds_listen was notified
Sep 22 23:21:37.783 DEBG up_ds_listen process 1000
Sep 22 23:21:37.783 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.783 INFO Upstairs starts
Sep 22 23:21:37.783 INFO Connection request from c3d60856-e346-48b1-a11b-e1e0e826a086 with version 4, task: proc
Sep 22 23:21:37.783 INFO upstairs UpstairsConnection { upstairs_id: c3d60856-e346-48b1-a11b-e1e0e826a086, session_id: c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.783 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.783 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.783 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.783 INFO Crucible stats registered with UUID: 0509f38e-fad6-4194-ae26-f925e684ff33
Sep 22 23:21:37.783 INFO Connection request from c3d60856-e346-48b1-a11b-e1e0e826a086 with version 4, task: proc
Sep 22 23:21:37.783 INFO Crucible 0509f38e-fad6-4194-ae26-f925e684ff33 has session id: d1777d3f-1d8b-4f8a-8fb7-855b4a3eb57a
Sep 22 23:21:37.783 INFO upstairs UpstairsConnection { upstairs_id: c3d60856-e346-48b1-a11b-e1e0e826a086, session_id: c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.783 INFO Connection request from c3d60856-e346-48b1-a11b-e1e0e826a086 with version 4, task: proc
Sep 22 23:21:37.783 INFO upstairs UpstairsConnection { upstairs_id: c3d60856-e346-48b1-a11b-e1e0e826a086, session_id: c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.783 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.783 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.783 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.783 INFO Created new region file "/tmp/downstairs-SCrOyIq5/region.json"
Sep 22 23:21:37.783 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.783 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.783 INFO [0] c3d60856-e346-48b1-a11b-e1e0e826a086 (c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b) New New New ds_transition to WaitActive
Sep 22 23:21:37.783 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.783 INFO [0] connecting to 127.0.0.1:60322, looper: 0
Sep 22 23:21:37.783 INFO [1] c3d60856-e346-48b1-a11b-e1e0e826a086 (c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.783 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.783 INFO [2] c3d60856-e346-48b1-a11b-e1e0e826a086 (c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.783 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.783 INFO [1] connecting to 127.0.0.1:48145, looper: 1
The guest has requested activation
Sep 22 23:21:37.783 INFO c3d60856-e346-48b1-a11b-e1e0e826a086 active request set
Sep 22 23:21:37.783 INFO [2] connecting to 127.0.0.1:40738, looper: 2
Sep 22 23:21:37.783 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.783 INFO Opened existing region file "/tmp/downstairs-4bgragna/region.json"
Sep 22 23:21:37.783 INFO Database read version 1
Sep 22 23:21:37.783 INFO [0] received activate with gen 1
Sep 22 23:21:37.783 INFO Database write version 1
Sep 22 23:21:37.783 INFO [0] client got ds_active_rx, promote! session c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b
Sep 22 23:21:37.783 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.783 INFO [1] received activate with gen 1
Sep 22 23:21:37.784 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.784 INFO [1] client got ds_active_rx, promote! session c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b
Sep 22 23:21:37.784 INFO Flush timeout: 0.5
Sep 22 23:21:37.784 INFO [2] received activate with gen 1
Sep 22 23:21:37.784 INFO [2] client got ds_active_rx, promote! session c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b
Sep 22 23:21:37.784 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.784 INFO UpstairsConnection { upstairs_id: c3d60856-e346-48b1-a11b-e1e0e826a086, session_id: c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b, gen: 1 } is now active (read-write)
Sep 22 23:21:37.784 INFO accepted connection from 127.0.0.1:45137, task: main
Sep 22 23:21:37.784 INFO UpstairsConnection { upstairs_id: c3d60856-e346-48b1-a11b-e1e0e826a086, session_id: c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b, gen: 1 } is now active (read-write)
Sep 22 23:21:37.784 INFO accepted connection from 127.0.0.1:36406, task: main
Sep 22 23:21:37.784 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.784 INFO UpstairsConnection { upstairs_id: c3d60856-e346-48b1-a11b-e1e0e826a086, session_id: c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b, gen: 1 } is now active (read-write)
Sep 22 23:21:37.784 INFO accepted connection from 127.0.0.1:46801, task: main
Sep 22 23:21:37.784 INFO [0] 0509f38e-fad6-4194-ae26-f925e684ff33 looper connected, looper: 0
Sep 22 23:21:37.784 INFO [0] Proc runs for 127.0.0.1:60322 in state New
Sep 22 23:21:37.784 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.784 INFO [1] 0509f38e-fad6-4194-ae26-f925e684ff33 looper connected, looper: 1
Sep 22 23:21:37.784 INFO [1] Proc runs for 127.0.0.1:48145 in state New
Sep 22 23:21:37.784 INFO [2] 0509f38e-fad6-4194-ae26-f925e684ff33 looper connected, looper: 2
Sep 22 23:21:37.784 INFO [0] downstairs client at 127.0.0.1:63732 has UUID 948c8c95-e366-4ab4-b43c-f85926d90cc9
Sep 22 23:21:37.784 INFO [2] Proc runs for 127.0.0.1:40738 in state New
Sep 22 23:21:37.784 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 948c8c95-e366-4ab4-b43c-f85926d90cc9, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.784 INFO c3d60856-e346-48b1-a11b-e1e0e826a086 WaitActive WaitActive WaitActive
Sep 22 23:21:37.784 INFO [1] downstairs client at 127.0.0.1:61215 has UUID d6f117a0-d707-4b2e-a491-e44ab08fd29f
Sep 22 23:21:37.784 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d6f117a0-d707-4b2e-a491-e44ab08fd29f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.784 INFO UUID: 8ac2d079-494b-4dc0-bfb3-1771389b3218
Sep 22 23:21:37.784 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.784 INFO c3d60856-e346-48b1-a11b-e1e0e826a086 WaitActive WaitActive WaitActive
Sep 22 23:21:37.784 DEBG [2] Read AckReady 1000, : downstairs
Sep 22 23:21:37.784 INFO [2] downstairs client at 127.0.0.1:62024 has UUID 3ab28074-4b58-4686-8a29-dc39469e56d5
Sep 22 23:21:37.784 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.784 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3ab28074-4b58-4686-8a29-dc39469e56d5, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.784 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.784 DEBG [0] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.784 INFO Using address: 127.0.0.1:64153, task: main
Sep 22 23:21:37.784 INFO c3d60856-e346-48b1-a11b-e1e0e826a086 WaitActive WaitActive WaitActive
Sep 22 23:21:37.785 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.785 DEBG up_ds_listen was notified
Sep 22 23:21:37.785 DEBG up_ds_listen process 1000
Sep 22 23:21:37.785 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.785 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.785 INFO Connection request from 0509f38e-fad6-4194-ae26-f925e684ff33 with version 4, task: proc
Sep 22 23:21:37.785 INFO upstairs UpstairsConnection { upstairs_id: 0509f38e-fad6-4194-ae26-f925e684ff33, session_id: 2e9b507b-029a-450c-8cce-275488d4eebe, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.785 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.785 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.785 INFO Connection request from 0509f38e-fad6-4194-ae26-f925e684ff33 with version 4, task: proc
Sep 22 23:21:37.785 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.785 INFO upstairs UpstairsConnection { upstairs_id: 0509f38e-fad6-4194-ae26-f925e684ff33, session_id: 2e9b507b-029a-450c-8cce-275488d4eebe, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.785 INFO Connection request from 0509f38e-fad6-4194-ae26-f925e684ff33 with version 4, task: proc
Sep 22 23:21:37.785 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53793, task: repair
Sep 22 23:21:37.785 INFO upstairs UpstairsConnection { upstairs_id: 0509f38e-fad6-4194-ae26-f925e684ff33, session_id: 2e9b507b-029a-450c-8cce-275488d4eebe, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.785 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53793, task: repair
Sep 22 23:21:37.785 INFO listening, local_addr: 127.0.0.1:53793, task: repair
The guest has requested activation
Sep 22 23:21:37.785 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.785 INFO [0] 0509f38e-fad6-4194-ae26-f925e684ff33 (2e9b507b-029a-450c-8cce-275488d4eebe) New New New ds_transition to WaitActive
Sep 22 23:21:37.785 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.785 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53793, task: repair
Sep 22 23:21:37.785 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.785 INFO Using repair address: 127.0.0.1:53793, task: main
Sep 22 23:21:37.785 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.785 INFO [1] 0509f38e-fad6-4194-ae26-f925e684ff33 (2e9b507b-029a-450c-8cce-275488d4eebe) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.785 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.785 INFO [2] 0509f38e-fad6-4194-ae26-f925e684ff33 (2e9b507b-029a-450c-8cce-275488d4eebe) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.785 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.785 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.785 INFO 0509f38e-fad6-4194-ae26-f925e684ff33 active request set
Sep 22 23:21:37.786 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.786 INFO [0] received activate with gen 1
Sep 22 23:21:37.786 INFO [0] client got ds_active_rx, promote! session 2e9b507b-029a-450c-8cce-275488d4eebe
Sep 22 23:21:37.786 INFO [0] c3d60856-e346-48b1-a11b-e1e0e826a086 (c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.786 INFO [1] received activate with gen 1
Sep 22 23:21:37.786 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.786 WARN [0] new RM replaced this: None
Sep 22 23:21:37.786 INFO [1] client got ds_active_rx, promote! session 2e9b507b-029a-450c-8cce-275488d4eebe
Sep 22 23:21:37.786 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.786 INFO [0] Starts reconcile loop
Sep 22 23:21:37.786 INFO [2] received activate with gen 1
Sep 22 23:21:37.786 INFO [1] c3d60856-e346-48b1-a11b-e1e0e826a086 (c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.786 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.786 INFO [2] client got ds_active_rx, promote! session 2e9b507b-029a-450c-8cce-275488d4eebe
Sep 22 23:21:37.786 WARN [1] new RM replaced this: None
Sep 22 23:21:37.786 INFO [1] Starts reconcile loop
Sep 22 23:21:37.786 INFO Created new region file "/tmp/downstairs-KCgrRonK/region.json"
Sep 22 23:21:37.786 INFO [2] c3d60856-e346-48b1-a11b-e1e0e826a086 (c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.786 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.786 WARN [2] new RM replaced this: None
Sep 22 23:21:37.786 INFO [2] Starts reconcile loop
Sep 22 23:21:37.786 INFO UpstairsConnection { upstairs_id: 0509f38e-fad6-4194-ae26-f925e684ff33, session_id: 2e9b507b-029a-450c-8cce-275488d4eebe, gen: 1 } is now active (read-write)
Sep 22 23:21:37.786 INFO [0] 127.0.0.1:63732 task reports connection:true
Sep 22 23:21:37.786 INFO c3d60856-e346-48b1-a11b-e1e0e826a086 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.786 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.786 INFO UUID: ed2b689d-1a8b-49a7-ad7b-f6874511d0ef
Sep 22 23:21:37.786 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.786 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.786 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.786 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.786 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.786 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.786 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.786 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.786 INFO UpstairsConnection { upstairs_id: 0509f38e-fad6-4194-ae26-f925e684ff33, session_id: 2e9b507b-029a-450c-8cce-275488d4eebe, gen: 1 } is now active (read-write)
Sep 22 23:21:37.786 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.786 INFO Max found gen is 1
Sep 22 23:21:37.786 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.786 INFO Next flush: 1
Sep 22 23:21:37.786 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.786 INFO All extents match
Sep 22 23:21:37.786 INFO No downstairs repair required
Sep 22 23:21:37.786 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.786 INFO No initial repair work was required
Sep 22 23:21:37.786 INFO Using address: 127.0.0.1:33453, task: main
Sep 22 23:21:37.786 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.786 INFO UpstairsConnection { upstairs_id: 0509f38e-fad6-4194-ae26-f925e684ff33, session_id: 2e9b507b-029a-450c-8cce-275488d4eebe, gen: 1 } is now active (read-write)
Sep 22 23:21:37.786 INFO c3d60856-e346-48b1-a11b-e1e0e826a086 is now active with session: c1e8da4a-6bc0-4db5-aac9-c4eb133e3c5b
Sep 22 23:21:37.786 INFO c3d60856-e346-48b1-a11b-e1e0e826a086 Set Active after no repair
Sep 22 23:21:37.786 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.786 INFO Set check for repair
Sep 22 23:21:37.786 INFO [1] 127.0.0.1:61215 task reports connection:true
Sep 22 23:21:37.786 INFO c3d60856-e346-48b1-a11b-e1e0e826a086 Active Active Active
Sep 22 23:21:37.786 INFO Set check for repair
Sep 22 23:21:37.786 INFO [2] 127.0.0.1:62024 task reports connection:true
Sep 22 23:21:37.786 INFO c3d60856-e346-48b1-a11b-e1e0e826a086 Active Active Active
Sep 22 23:21:37.786 INFO Set check for repair
Sep 22 23:21:37.786 INFO [0] received reconcile message
Sep 22 23:21:37.787 INFO [0] All repairs completed, exit
Sep 22 23:21:37.787 INFO [0] Starts cmd_loop
Sep 22 23:21:37.787 INFO [1] received reconcile message
Sep 22 23:21:37.787 INFO [1] All repairs completed, exit
Sep 22 23:21:37.787 INFO [1] Starts cmd_loop
Sep 22 23:21:37.787 INFO [2] received reconcile message
Sep 22 23:21:37.787 INFO [2] All repairs completed, exit
Sep 22 23:21:37.787 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:37.787 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.787 INFO [0] downstairs client at 127.0.0.1:60322 has UUID a6815f0f-5f43-4904-a056-e79c5267f195
Sep 22 23:21:37.787 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a6815f0f-5f43-4904-a056-e79c5267f195, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.787 INFO 0509f38e-fad6-4194-ae26-f925e684ff33 WaitActive WaitActive WaitActive
Sep 22 23:21:37.787 INFO [1] downstairs client at 127.0.0.1:48145 has UUID 9d3f8a28-aea4-4f16-a02f-499f4731649b
Sep 22 23:21:37.787 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52333, task: repair
Sep 22 23:21:37.787 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9d3f8a28-aea4-4f16-a02f-499f4731649b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.787 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52333, task: repair
Sep 22 23:21:37.787 INFO 0509f38e-fad6-4194-ae26-f925e684ff33 WaitActive WaitActive WaitActive
Sep 22 23:21:37.787 INFO listening, local_addr: 127.0.0.1:52333, task: repair
Sep 22 23:21:37.787 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.787 INFO [2] downstairs client at 127.0.0.1:40738 has UUID 890de4ee-4d21-427c-8570-1ce5a5506828
Sep 22 23:21:37.787 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 890de4ee-4d21-427c-8570-1ce5a5506828, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.787 INFO 0509f38e-fad6-4194-ae26-f925e684ff33 WaitActive WaitActive WaitActive
Sep 22 23:21:37.787 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52333, task: repair
Sep 22 23:21:37.787 INFO Using repair address: 127.0.0.1:52333, task: main
Sep 22 23:21:37.787 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.787 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.788 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.788 INFO Opened existing region file "/tmp/downstairs-SCrOyIq5/region.json"
Sep 22 23:21:37.788 INFO Database read version 1
Sep 22 23:21:37.788 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.788 INFO Database write version 1
Sep 22 23:21:37.788 INFO Created new region file "/tmp/downstairs-ZkhS3bbX/region.json"
Sep 22 23:21:37.788 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.788 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.789 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.789 DEBG Read      :1000 deps:[] res:true
test test::integration_test_multi_read_only ... ok
Sep 22 23:21:37.789 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.789 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.789 INFO Created new region file "/tmp/downstairs-VBH4uE22/region.json"
Sep 22 23:21:37.789 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.789 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.790 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.790 INFO [0] 0509f38e-fad6-4194-ae26-f925e684ff33 (2e9b507b-029a-450c-8cce-275488d4eebe) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.790 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.790 WARN [0] new RM replaced this: None
Sep 22 23:21:37.790 INFO UUID: bdfd9105-b870-40c3-a737-5c05b9c9e488
Sep 22 23:21:37.790 INFO [0] Starts reconcile loop
Sep 22 23:21:37.790 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.790 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.790 INFO [1] 0509f38e-fad6-4194-ae26-f925e684ff33 (2e9b507b-029a-450c-8cce-275488d4eebe) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.790 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.790 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.790 INFO Using address: 127.0.0.1:50238, task: main
Sep 22 23:21:37.790 WARN [1] new RM replaced this: None
Sep 22 23:21:37.790 INFO [1] Starts reconcile loop
Sep 22 23:21:37.790 INFO [2] 0509f38e-fad6-4194-ae26-f925e684ff33 (2e9b507b-029a-450c-8cce-275488d4eebe) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.790 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.790 WARN [2] new RM replaced this: None
Sep 22 23:21:37.790 INFO [2] Starts reconcile loop
Sep 22 23:21:37.790 INFO [0] 127.0.0.1:60322 task reports connection:true
Sep 22 23:21:37.790 INFO 0509f38e-fad6-4194-ae26-f925e684ff33 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.790 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.790 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.790 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.790 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.790 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.790 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.790 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.790 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38527, task: repair
Sep 22 23:21:37.790 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.790 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.790 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.790 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38527, task: repair
Sep 22 23:21:37.790 INFO Max found gen is 1
Sep 22 23:21:37.790 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.790 INFO listening, local_addr: 127.0.0.1:38527, task: repair
Sep 22 23:21:37.791 INFO Next flush: 1
Sep 22 23:21:37.791 INFO All extents match
Sep 22 23:21:37.791 INFO No downstairs repair required
Sep 22 23:21:37.791 INFO No initial repair work was required
Sep 22 23:21:37.791 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.791 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.791 INFO 0509f38e-fad6-4194-ae26-f925e684ff33 is now active with session: 2e9b507b-029a-450c-8cce-275488d4eebe
Sep 22 23:21:37.791 INFO 0509f38e-fad6-4194-ae26-f925e684ff33 Set Active after no repair
Sep 22 23:21:37.791 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.791 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38527, task: repair
Sep 22 23:21:37.791 INFO Set check for repair
Sep 22 23:21:37.791 INFO Using repair address: 127.0.0.1:38527, task: main
Sep 22 23:21:37.791 INFO [1] 127.0.0.1:48145 task reports connection:true
Sep 22 23:21:37.791 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.791 INFO 0509f38e-fad6-4194-ae26-f925e684ff33 Active Active Active
Sep 22 23:21:37.791 INFO Set check for repair
Sep 22 23:21:37.791 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.791 INFO [2] 127.0.0.1:40738 task reports connection:true
Sep 22 23:21:37.791 INFO 0509f38e-fad6-4194-ae26-f925e684ff33 Active Active Active
Sep 22 23:21:37.791 INFO Set check for repair
Sep 22 23:21:37.791 INFO [0] received reconcile message
Sep 22 23:21:37.791 INFO [0] All repairs completed, exit
Sep 22 23:21:37.791 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.791 INFO [0] Starts cmd_loop
Sep 22 23:21:37.791 DEBG up_ds_listen was notified
Sep 22 23:21:37.791 INFO [1] received reconcile message
Sep 22 23:21:37.791 DEBG up_ds_listen process 1000
Sep 22 23:21:37.791 INFO [1] All repairs completed, exit
Sep 22 23:21:37.791 INFO [1] Starts cmd_loop
Sep 22 23:21:37.791 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.791 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.791 INFO Opened existing region file "/tmp/downstairs-KCgrRonK/region.json"
Sep 22 23:21:37.791 INFO [2] received reconcile message
Sep 22 23:21:37.791 INFO Database read version 1
Sep 22 23:21:37.791 INFO Database write version 1
Sep 22 23:21:37.791 INFO [2] All repairs completed, exit
Sep 22 23:21:37.791 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:37.791 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.791 INFO Created new region file "/tmp/downstairs-Ezm5ouTm/region.json"
Sep 22 23:21:37.792 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.792 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.793 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.793 INFO Opened existing region file "/tmp/downstairs-ZkhS3bbX/region.json"
Sep 22 23:21:37.793 INFO Database read version 1
Sep 22 23:21:37.793 INFO Database write version 1
Sep 22 23:21:37.793 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.793 INFO UUID: 816d86cc-233d-41c9-991c-d98e4fdc6748
Sep 22 23:21:37.793 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.793 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.793 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.793 INFO Using address: 127.0.0.1:59973, task: main
Sep 22 23:21:37.793 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.794 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.794 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47256, task: repair
Sep 22 23:21:37.794 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47256, task: repair
Sep 22 23:21:37.794 INFO listening, local_addr: 127.0.0.1:47256, task: repair
Sep 22 23:21:37.794 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.794 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47256, task: repair
Sep 22 23:21:37.794 INFO Using repair address: 127.0.0.1:47256, task: main
Sep 22 23:21:37.794 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.794 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.794 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.795 INFO Opened existing region file "/tmp/downstairs-VBH4uE22/region.json"
Sep 22 23:21:37.795 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:37.795 INFO Database read version 1
Sep 22 23:21:37.795 INFO Database write version 1
Sep 22 23:21:37.795 INFO Created new region file "/tmp/downstairs-KV5pdWk6/region.json"
Sep 22 23:21:37.795 DEBG up_ds_listen was notified
Sep 22 23:21:37.795 DEBG up_ds_listen process 1001
Sep 22 23:21:37.795 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.795 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.795 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.795 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.795 INFO UUID: e251ad5f-6614-4ef0-b821-00d45953fcfc
Sep 22 23:21:37.795 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.796 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.796 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.796 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.796 INFO Using address: 127.0.0.1:36989, task: main
Sep 22 23:21:37.796 DEBG up_ds_listen was notified
Sep 22 23:21:37.796 DEBG up_ds_listen process 1000
Sep 22 23:21:37.796 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.796 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.796 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37445, task: repair
Sep 22 23:21:37.796 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37445, task: repair
Sep 22 23:21:37.796 INFO listening, local_addr: 127.0.0.1:37445, task: repair
Sep 22 23:21:37.796 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.796 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37445, task: repair
Sep 22 23:21:37.796 INFO Using repair address: 127.0.0.1:37445, task: main
Sep 22 23:21:37.796 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.797 INFO UUID: 225348bb-ea71-4678-9320-8b37dbfcd881
Sep 22 23:21:37.797 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.797 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.797 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.797 INFO Using address: 127.0.0.1:38477, task: main
Sep 22 23:21:37.797 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.797 INFO Created new region file "/tmp/downstairs-PbY62a19/region.json"
Sep 22 23:21:37.797 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.797 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50593, task: repair
Sep 22 23:21:37.797 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50593, task: repair
Sep 22 23:21:37.797 INFO listening, local_addr: 127.0.0.1:50593, task: repair
Sep 22 23:21:37.797 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50593, task: repair
Sep 22 23:21:37.797 INFO Using repair address: 127.0.0.1:50593, task: main
Sep 22 23:21:37.797 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.797 INFO Scrub check for 996ff2d1-184c-4c18-9e4b-609bb90feb01
Sep 22 23:21:37.798 INFO Scrub for 996ff2d1-184c-4c18-9e4b-609bb90feb01 begins
Sep 22 23:21:37.798 INFO Scrub with total_size:5120 block_size:512
Sep 22 23:21:37.798 INFO Scrubs from block 0 to 10 in (256) 131072 size IOs pm:0
Sep 22 23:21:37.798 INFO Adjust block_count to 10 at offset 0
Sep 22 23:21:37.798 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.798 INFO Opened existing region file "/tmp/downstairs-Ezm5ouTm/region.json"
Sep 22 23:21:37.798 INFO Database read version 1
Sep 22 23:21:37.798 INFO Database write version 1
Sep 22 23:21:37.798 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.798 INFO Created new region file "/tmp/downstairs-xy4dQl33/region.json"
Sep 22 23:21:37.798 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.799 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.799 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.799 INFO Opened existing region file "/tmp/downstairs-KV5pdWk6/region.json"
Sep 22 23:21:37.799 INFO Database read version 1
Sep 22 23:21:37.799 INFO Database write version 1
Sep 22 23:21:37.800 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.801 INFO UUID: 7f0f7567-102c-47f7-afe6-1eed37005958
Sep 22 23:21:37.801 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.801 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:21:37.801 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.801 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.801 INFO Using address: 127.0.0.1:64264, task: main
Sep 22 23:21:37.801 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:37.801 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.801 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33498, task: repair
Sep 22 23:21:37.801 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33498, task: repair
Sep 22 23:21:37.801 INFO listening, local_addr: 127.0.0.1:33498, task: repair
Sep 22 23:21:37.801 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.801 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33498, task: repair
Sep 22 23:21:37.801 INFO Using repair address: 127.0.0.1:33498, task: main
Sep 22 23:21:37.801 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.802 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.802 INFO Upstairs starts
Sep 22 23:21:37.802 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.802 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.802 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.802 INFO Crucible stats registered with UUID: c27ba65a-3c7e-4977-b88b-9c74b48e942f
Sep 22 23:21:37.802 INFO Crucible c27ba65a-3c7e-4977-b88b-9c74b48e942f has session id: 1e3ca93d-2e4e-478f-b33f-3a1688e22eed
Sep 22 23:21:37.802 INFO UUID: 8d7b21f8-00cc-4c41-81e9-de537a7d24ce
Sep 22 23:21:37.802 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.803 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.803 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.803 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.803 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.803 INFO Using address: 127.0.0.1:46588, task: main
Sep 22 23:21:37.803 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.803 INFO Opened existing region file "/tmp/downstairs-xy4dQl33/region.json"
Sep 22 23:21:37.803 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.803 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.803 INFO Database read version 1
Sep 22 23:21:37.803 INFO Database write version 1
Sep 22 23:21:37.803 INFO Opened existing region file "/tmp/downstairs-PbY62a19/region.json"
Sep 22 23:21:37.803 INFO Database read version 1
Sep 22 23:21:37.803 INFO Database write version 1
Sep 22 23:21:37.803 INFO [0] connecting to 127.0.0.1:49534, looper: 0
Sep 22 23:21:37.803 INFO [1] connecting to 127.0.0.1:50238, looper: 1
Sep 22 23:21:37.803 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.803 INFO [2] connecting to 127.0.0.1:64264, looper: 2
Sep 22 23:21:37.803 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48398, task: repair
Sep 22 23:21:37.803 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.803 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.803 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48398, task: repair
Sep 22 23:21:37.803 INFO Flush timeout: 0.5
Sep 22 23:21:37.803 INFO listening, local_addr: 127.0.0.1:48398, task: repair
Sep 22 23:21:37.803 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48398, task: repair
Sep 22 23:21:37.803 INFO Using repair address: 127.0.0.1:48398, task: main
Sep 22 23:21:37.803 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.803 INFO accepted connection from 127.0.0.1:63449, task: main
Sep 22 23:21:37.804 INFO accepted connection from 127.0.0.1:42043, task: main
Sep 22 23:21:37.804 INFO accepted connection from 127.0.0.1:62571, task: main
Sep 22 23:21:37.804 INFO [0] c27ba65a-3c7e-4977-b88b-9c74b48e942f looper connected, looper: 0
Sep 22 23:21:37.804 INFO [0] Proc runs for 127.0.0.1:49534 in state New
Sep 22 23:21:37.804 INFO [1] c27ba65a-3c7e-4977-b88b-9c74b48e942f looper connected, looper: 1
Sep 22 23:21:37.804 INFO [1] Proc runs for 127.0.0.1:50238 in state New
Sep 22 23:21:37.804 INFO [2] c27ba65a-3c7e-4977-b88b-9c74b48e942f looper connected, looper: 2
Sep 22 23:21:37.804 INFO [2] Proc runs for 127.0.0.1:64264 in state New
Sep 22 23:21:37.804 INFO Connection request from c27ba65a-3c7e-4977-b88b-9c74b48e942f with version 4, task: proc
Sep 22 23:21:37.804 INFO upstairs UpstairsConnection { upstairs_id: c27ba65a-3c7e-4977-b88b-9c74b48e942f, session_id: 046c4783-9946-4b0c-8eb5-0996fef4cd23, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.804 INFO Connection request from c27ba65a-3c7e-4977-b88b-9c74b48e942f with version 4, task: proc
Sep 22 23:21:37.805 INFO Upstairs starts
Sep 22 23:21:37.805 INFO upstairs UpstairsConnection { upstairs_id: c27ba65a-3c7e-4977-b88b-9c74b48e942f, session_id: 046c4783-9946-4b0c-8eb5-0996fef4cd23, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.805 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.805 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.805 INFO Crucible stats registered with UUID: 8a9244fd-a219-4407-95b8-2e8c6b37613e
Sep 22 23:21:37.805 INFO Crucible 8a9244fd-a219-4407-95b8-2e8c6b37613e has session id: 05c4a641-4bef-4a5b-bbfa-45eb16265cac
Sep 22 23:21:37.805 INFO Connection request from c27ba65a-3c7e-4977-b88b-9c74b48e942f with version 4, task: proc
Sep 22 23:21:37.805 INFO upstairs UpstairsConnection { upstairs_id: c27ba65a-3c7e-4977-b88b-9c74b48e942f, session_id: 046c4783-9946-4b0c-8eb5-0996fef4cd23, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:21:37.805 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.805 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.805 INFO [0] c27ba65a-3c7e-4977-b88b-9c74b48e942f (046c4783-9946-4b0c-8eb5-0996fef4cd23) New New New ds_transition to WaitActive
Sep 22 23:21:37.805 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.805 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.805 INFO [0] connecting to 127.0.0.1:64153, looper: 0
Sep 22 23:21:37.805 INFO [1] c27ba65a-3c7e-4977-b88b-9c74b48e942f (046c4783-9946-4b0c-8eb5-0996fef4cd23) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.805 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:21:37.805 INFO [1] connecting to 127.0.0.1:59973, looper: 1
Sep 22 23:21:37.805 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.805 INFO [2] c27ba65a-3c7e-4977-b88b-9c74b48e942f (046c4783-9946-4b0c-8eb5-0996fef4cd23) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.805 INFO [2] connecting to 127.0.0.1:46588, looper: 2
Sep 22 23:21:37.805 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.805 INFO c27ba65a-3c7e-4977-b88b-9c74b48e942f active request set
Sep 22 23:21:37.805 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.805 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.805 INFO Flush timeout: 0.5
Sep 22 23:21:37.805 INFO [0] received activate with gen 1
Sep 22 23:21:37.806 INFO [0] client got ds_active_rx, promote! session 046c4783-9946-4b0c-8eb5-0996fef4cd23
Sep 22 23:21:37.806 INFO [1] received activate with gen 1
Sep 22 23:21:37.806 INFO [1] client got ds_active_rx, promote! session 046c4783-9946-4b0c-8eb5-0996fef4cd23
Sep 22 23:21:37.806 INFO [2] received activate with gen 1
Sep 22 23:21:37.806 INFO accepted connection from 127.0.0.1:58780, task: main
Sep 22 23:21:37.806 INFO [2] client got ds_active_rx, promote! session 046c4783-9946-4b0c-8eb5-0996fef4cd23
Sep 22 23:21:37.806 INFO accepted connection from 127.0.0.1:46822, task: main
Sep 22 23:21:37.806 INFO UUID: 2a37b56b-50b2-4d7c-9cf7-bf9557fd6093
Sep 22 23:21:37.806 INFO UUID: e05c70a3-48dd-4fb2-827c-e28061a0ccac
Sep 22 23:21:37.806 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.806 INFO accepted connection from 127.0.0.1:57952, task: main
Sep 22 23:21:37.806 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.806 INFO UpstairsConnection { upstairs_id: c27ba65a-3c7e-4977-b88b-9c74b48e942f, session_id: 046c4783-9946-4b0c-8eb5-0996fef4cd23, gen: 1 } is now active (read-write)
Sep 22 23:21:37.806 INFO [0] 8a9244fd-a219-4407-95b8-2e8c6b37613e looper connected, looper: 0
Sep 22 23:21:37.806 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.806 INFO [0] Proc runs for 127.0.0.1:64153 in state New
Sep 22 23:21:37.806 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.806 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.806 INFO UpstairsConnection { upstairs_id: c27ba65a-3c7e-4977-b88b-9c74b48e942f, session_id: 046c4783-9946-4b0c-8eb5-0996fef4cd23, gen: 1 } is now active (read-write)
Sep 22 23:21:37.806 INFO Using address: 127.0.0.1:39308, task: main
Sep 22 23:21:37.806 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.806 INFO Using address: 127.0.0.1:55757, task: main
Sep 22 23:21:37.806 DEBG up_ds_listen was notified
Sep 22 23:21:37.806 INFO [1] 8a9244fd-a219-4407-95b8-2e8c6b37613e looper connected, looper: 1
Sep 22 23:21:37.806 DEBG up_ds_listen process 1001
Sep 22 23:21:37.806 INFO [1] Proc runs for 127.0.0.1:59973 in state New
Sep 22 23:21:37.806 INFO UpstairsConnection { upstairs_id: c27ba65a-3c7e-4977-b88b-9c74b48e942f, session_id: 046c4783-9946-4b0c-8eb5-0996fef4cd23, gen: 1 } is now active (read-write)
Sep 22 23:21:37.806 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.806 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.806 INFO [2] 8a9244fd-a219-4407-95b8-2e8c6b37613e looper connected, looper: 2
Sep 22 23:21:37.806 INFO [2] Proc runs for 127.0.0.1:46588 in state New
Sep 22 23:21:37.806 INFO Scrub at offset 10/10 sp:10
Sep 22 23:21:37.806 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.806 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.806 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:45242, task: repair
Sep 22 23:21:37.806 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:45242, task: repair
Sep 22 23:21:37.806 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:59077, task: repair
Sep 22 23:21:37.807 INFO listening, local_addr: 127.0.0.1:45242, task: repair
Sep 22 23:21:37.807 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:59077, task: repair
Sep 22 23:21:37.807 INFO Connection request from 8a9244fd-a219-4407-95b8-2e8c6b37613e with version 4, task: proc
Sep 22 23:21:37.807 INFO listening, local_addr: 127.0.0.1:59077, task: repair
Sep 22 23:21:37.807 INFO upstairs UpstairsConnection { upstairs_id: 8a9244fd-a219-4407-95b8-2e8c6b37613e, session_id: af1123f4-626a-46f8-a46e-4008ec56e922, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.807 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:45242, task: repair
Sep 22 23:21:37.807 INFO Using repair address: 127.0.0.1:45242, task: main
Sep 22 23:21:37.807 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.807 INFO [0] downstairs client at 127.0.0.1:49534 has UUID 8f0829a0-9286-4151-a390-4aadd7c97e80
Sep 22 23:21:37.807 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 8f0829a0-9286-4151-a390-4aadd7c97e80, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.807 INFO Connection request from 8a9244fd-a219-4407-95b8-2e8c6b37613e with version 4, task: proc
Sep 22 23:21:37.807 INFO upstairs UpstairsConnection { upstairs_id: 8a9244fd-a219-4407-95b8-2e8c6b37613e, session_id: af1123f4-626a-46f8-a46e-4008ec56e922, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.807 INFO c27ba65a-3c7e-4977-b88b-9c74b48e942f WaitActive WaitActive WaitActive
Sep 22 23:21:37.807 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:59077, task: repair
Sep 22 23:21:37.807 INFO Using repair address: 127.0.0.1:59077, task: main
Sep 22 23:21:37.807 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.807 INFO [1] downstairs client at 127.0.0.1:50238 has UUID bdfd9105-b870-40c3-a737-5c05b9c9e488
Sep 22 23:21:37.807 INFO Connection request from 8a9244fd-a219-4407-95b8-2e8c6b37613e with version 4, task: proc
Sep 22 23:21:37.807 INFO upstairs UpstairsConnection { upstairs_id: 8a9244fd-a219-4407-95b8-2e8c6b37613e, session_id: af1123f4-626a-46f8-a46e-4008ec56e922, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.807 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: bdfd9105-b870-40c3-a737-5c05b9c9e488, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.807 INFO c27ba65a-3c7e-4977-b88b-9c74b48e942f WaitActive WaitActive WaitActive
Sep 22 23:21:37.807 INFO [2] downstairs client at 127.0.0.1:64264 has UUID 7f0f7567-102c-47f7-afe6-1eed37005958
Sep 22 23:21:37.807 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.807 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7f0f7567-102c-47f7-afe6-1eed37005958, encrypted: true, database_read_version: 1, database_write_version: 1 }
The guest has requested activation
Sep 22 23:21:37.807 INFO c27ba65a-3c7e-4977-b88b-9c74b48e942f WaitActive WaitActive WaitActive
Sep 22 23:21:37.807 INFO Upstairs starts
Sep 22 23:21:37.807 INFO [0] 8a9244fd-a219-4407-95b8-2e8c6b37613e (af1123f4-626a-46f8-a46e-4008ec56e922) New New New ds_transition to WaitActive
Sep 22 23:21:37.807 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.807 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.807 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.807 INFO Crucible stats registered with UUID: bc449dbb-4e8d-49d4-9ec4-c4ddf333de98
Sep 22 23:21:37.807 INFO Crucible bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 has session id: 44632fdd-5827-4865-829c-ea82538a3d81
Sep 22 23:21:37.807 INFO [1] 8a9244fd-a219-4407-95b8-2e8c6b37613e (af1123f4-626a-46f8-a46e-4008ec56e922) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.807 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.807 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.807 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.807 INFO [2] 8a9244fd-a219-4407-95b8-2e8c6b37613e (af1123f4-626a-46f8-a46e-4008ec56e922) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.807 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.807 INFO Created new region file "/tmp/downstairs-iBpIXgok/region.json"
Sep 22 23:21:37.807 INFO 8a9244fd-a219-4407-95b8-2e8c6b37613e active request set
Sep 22 23:21:37.807 INFO Scrub 996ff2d1-184c-4c18-9e4b-609bb90feb01 done in 0 seconds. Retries:0 scrub_size:5120 size:10 pause_milli:0
Sep 22 23:21:37.807 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.808 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.808 INFO [0] received activate with gen 1
Sep 22 23:21:37.808 INFO [0] client got ds_active_rx, promote! session af1123f4-626a-46f8-a46e-4008ec56e922
Sep 22 23:21:37.808 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.808 DEBG IO Flush 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:37.808 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.808 INFO [1] received activate with gen 1
Sep 22 23:21:37.808 INFO [1] client got ds_active_rx, promote! session af1123f4-626a-46f8-a46e-4008ec56e922
Sep 22 23:21:37.808 INFO [0] connecting to 127.0.0.1:33453, looper: 0
Sep 22 23:21:37.808 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.808 INFO [2] received activate with gen 1
Sep 22 23:21:37.808 INFO [2] client got ds_active_rx, promote! session af1123f4-626a-46f8-a46e-4008ec56e922
Sep 22 23:21:37.808 INFO [1] connecting to 127.0.0.1:36989, looper: 1
Sep 22 23:21:37.808 INFO UpstairsConnection { upstairs_id: 8a9244fd-a219-4407-95b8-2e8c6b37613e, session_id: af1123f4-626a-46f8-a46e-4008ec56e922, gen: 1 } is now active (read-write)
Sep 22 23:21:37.808 INFO [2] connecting to 127.0.0.1:55757, looper: 2
Sep 22 23:21:37.808 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.808 INFO UpstairsConnection { upstairs_id: 8a9244fd-a219-4407-95b8-2e8c6b37613e, session_id: af1123f4-626a-46f8-a46e-4008ec56e922, gen: 1 } is now active (read-write)
Sep 22 23:21:37.808 INFO UpstairsConnection { upstairs_id: 8a9244fd-a219-4407-95b8-2e8c6b37613e, session_id: af1123f4-626a-46f8-a46e-4008ec56e922, gen: 1 } is now active (read-write)
Sep 22 23:21:37.808 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.808 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.808 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.808 INFO Flush timeout: 0.5
Sep 22 23:21:37.808 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.809 INFO [0] c27ba65a-3c7e-4977-b88b-9c74b48e942f (046c4783-9946-4b0c-8eb5-0996fef4cd23) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.809 INFO accepted connection from 127.0.0.1:37528, task: main
Sep 22 23:21:37.809 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.809 WARN [0] new RM replaced this: None
Sep 22 23:21:37.809 INFO [0] downstairs client at 127.0.0.1:64153 has UUID 8ac2d079-494b-4dc0-bfb3-1771389b3218
Sep 22 23:21:37.809 INFO [0] Starts reconcile loop
Sep 22 23:21:37.809 INFO accepted connection from 127.0.0.1:39885, task: main
Sep 22 23:21:37.809 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 8ac2d079-494b-4dc0-bfb3-1771389b3218, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.809 INFO 8a9244fd-a219-4407-95b8-2e8c6b37613e WaitActive WaitActive WaitActive
Sep 22 23:21:37.809 INFO [0] bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 looper connected, looper: 0
Sep 22 23:21:37.809 INFO [1] c27ba65a-3c7e-4977-b88b-9c74b48e942f (046c4783-9946-4b0c-8eb5-0996fef4cd23) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.809 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.809 INFO [0] Proc runs for 127.0.0.1:33453 in state New
Sep 22 23:21:37.809 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.809 WARN [1] new RM replaced this: None
Sep 22 23:21:37.809 INFO [1] downstairs client at 127.0.0.1:59973 has UUID 816d86cc-233d-41c9-991c-d98e4fdc6748
Sep 22 23:21:37.809 INFO [1] Starts reconcile loop
Sep 22 23:21:37.809 DEBG up_ds_listen was notified
Sep 22 23:21:37.809 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 816d86cc-233d-41c9-991c-d98e4fdc6748, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.809 DEBG up_ds_listen process 1002
Sep 22 23:21:37.809 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.809 INFO accepted connection from 127.0.0.1:43288, task: main
Sep 22 23:21:37.809 INFO 8a9244fd-a219-4407-95b8-2e8c6b37613e WaitActive WaitActive WaitActive
Sep 22 23:21:37.809 INFO [2] c27ba65a-3c7e-4977-b88b-9c74b48e942f (046c4783-9946-4b0c-8eb5-0996fef4cd23) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.809 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.809 WARN [2] new RM replaced this: None
Sep 22 23:21:37.809 INFO [2] bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 looper connected, looper: 2
Sep 22 23:21:37.809 INFO [2] Starts reconcile loop
Sep 22 23:21:37.809 INFO [2] downstairs client at 127.0.0.1:46588 has UUID 8d7b21f8-00cc-4c41-81e9-de537a7d24ce
Sep 22 23:21:37.809 INFO [2] Proc runs for 127.0.0.1:55757 in state New
Sep 22 23:21:37.809 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 8d7b21f8-00cc-4c41-81e9-de537a7d24ce, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.809 INFO [0] 127.0.0.1:49534 task reports connection:true
Sep 22 23:21:37.809 INFO 8a9244fd-a219-4407-95b8-2e8c6b37613e WaitActive WaitActive WaitActive
Sep 22 23:21:37.809 INFO c27ba65a-3c7e-4977-b88b-9c74b48e942f WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.809 INFO [1] bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 looper connected, looper: 1
Sep 22 23:21:37.809 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.809 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.809 INFO [1] Proc runs for 127.0.0.1:36989 in state New
Sep 22 23:21:37.809 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.809 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.809 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.809 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.809 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.809 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.809 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.809 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.809 INFO Max found gen is 1
Sep 22 23:21:37.809 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.809 INFO Next flush: 1
Sep 22 23:21:37.809 INFO All extents match
Sep 22 23:21:37.809 INFO No downstairs repair required
Sep 22 23:21:37.809 INFO No initial repair work was required
Sep 22 23:21:37.809 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.809 INFO c27ba65a-3c7e-4977-b88b-9c74b48e942f is now active with session: 046c4783-9946-4b0c-8eb5-0996fef4cd23
Sep 22 23:21:37.809 INFO c27ba65a-3c7e-4977-b88b-9c74b48e942f Set Active after no repair
Sep 22 23:21:37.809 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.809 INFO Set check for repair
Sep 22 23:21:37.809 INFO [1] 127.0.0.1:50238 task reports connection:true
Sep 22 23:21:37.810 INFO c27ba65a-3c7e-4977-b88b-9c74b48e942f Active Active Active
Sep 22 23:21:37.810 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.810 INFO Set check for repair
Sep 22 23:21:37.810 INFO [2] 127.0.0.1:64264 task reports connection:true
Sep 22 23:21:37.810 INFO c27ba65a-3c7e-4977-b88b-9c74b48e942f Active Active Active
Sep 22 23:21:37.810 INFO Connection request from bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 with version 4, task: proc
Sep 22 23:21:37.810 INFO Set check for repair
Sep 22 23:21:37.810 INFO upstairs UpstairsConnection { upstairs_id: bc449dbb-4e8d-49d4-9ec4-c4ddf333de98, session_id: 82d82a34-37ca-41ad-83f4-a1bf54c18aff, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.810 INFO [0] received reconcile message
Sep 22 23:21:37.810 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.810 INFO [0] All repairs completed, exit
Sep 22 23:21:37.810 INFO Connection request from bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 with version 4, task: proc
Sep 22 23:21:37.810 INFO [0] Starts cmd_loop
Sep 22 23:21:37.810 INFO upstairs UpstairsConnection { upstairs_id: bc449dbb-4e8d-49d4-9ec4-c4ddf333de98, session_id: 82d82a34-37ca-41ad-83f4-a1bf54c18aff, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.810 INFO [1] received reconcile message
Sep 22 23:21:37.810 INFO Connection request from bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 with version 4, task: proc
Sep 22 23:21:37.810 INFO [1] All repairs completed, exit
Sep 22 23:21:37.810 INFO upstairs UpstairsConnection { upstairs_id: bc449dbb-4e8d-49d4-9ec4-c4ddf333de98, session_id: 82d82a34-37ca-41ad-83f4-a1bf54c18aff, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.810 INFO [1] Starts cmd_loop
Sep 22 23:21:37.810 INFO [2] received reconcile message
Sep 22 23:21:37.810 INFO [2] All repairs completed, exit
Sep 22 23:21:37.810 INFO [2] Starts cmd_loop
The guest has requested activation
The guest has finished waiting for activation
Sep 22 23:21:37.810 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.810 INFO [0] bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 (82d82a34-37ca-41ad-83f4-a1bf54c18aff) New New New ds_transition to WaitActive
Sep 22 23:21:37.810 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.810 INFO [2] bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 (82d82a34-37ca-41ad-83f4-a1bf54c18aff) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.810 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.810 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.810 INFO [1] bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 (82d82a34-37ca-41ad-83f4-a1bf54c18aff) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:21:37.810 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.810 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.810 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.810 INFO bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 active request set
Sep 22 23:21:37.811 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.811 INFO [0] received activate with gen 1
Sep 22 23:21:37.811 INFO [0] client got ds_active_rx, promote! session 82d82a34-37ca-41ad-83f4-a1bf54c18aff
Sep 22 23:21:37.811 INFO [1] received activate with gen 1
Sep 22 23:21:37.811 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.811 INFO [1] client got ds_active_rx, promote! session 82d82a34-37ca-41ad-83f4-a1bf54c18aff
Sep 22 23:21:37.811 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.811 INFO [2] received activate with gen 1
Sep 22 23:21:37.811 INFO [2] client got ds_active_rx, promote! session 82d82a34-37ca-41ad-83f4-a1bf54c18aff
Sep 22 23:21:37.811 INFO UpstairsConnection { upstairs_id: bc449dbb-4e8d-49d4-9ec4-c4ddf333de98, session_id: 82d82a34-37ca-41ad-83f4-a1bf54c18aff, gen: 1 } is now active (read-write)
Sep 22 23:21:37.811 DEBG up_ds_listen was notified
Sep 22 23:21:37.811 INFO UpstairsConnection { upstairs_id: bc449dbb-4e8d-49d4-9ec4-c4ddf333de98, session_id: 82d82a34-37ca-41ad-83f4-a1bf54c18aff, gen: 1 } is now active (read-write)
Sep 22 23:21:37.811 DEBG up_ds_listen process 1002
Sep 22 23:21:37.811 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.811 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.811 INFO Opened existing region file "/tmp/downstairs-iBpIXgok/region.json"
Sep 22 23:21:37.811 INFO Database read version 1
Sep 22 23:21:37.811 INFO Database write version 1
Sep 22 23:21:37.811 INFO UpstairsConnection { upstairs_id: bc449dbb-4e8d-49d4-9ec4-c4ddf333de98, session_id: 82d82a34-37ca-41ad-83f4-a1bf54c18aff, gen: 1 } is now active (read-write)
Sep 22 23:21:37.811 DEBG [rc] retire 1002 clears [JobId(1000), JobId(1001), JobId(1002)], : downstairs
Sep 22 23:21:37.811 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.811 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.812 INFO [0] 8a9244fd-a219-4407-95b8-2e8c6b37613e (af1123f4-626a-46f8-a46e-4008ec56e922) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.812 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.812 WARN [0] new RM replaced this: None
Sep 22 23:21:37.812 INFO [0] Starts reconcile loop
Sep 22 23:21:37.812 INFO [0] downstairs client at 127.0.0.1:33453 has UUID ed2b689d-1a8b-49a7-ad7b-f6874511d0ef
Sep 22 23:21:37.812 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ed2b689d-1a8b-49a7-ad7b-f6874511d0ef, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.812 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.812 INFO [1] 8a9244fd-a219-4407-95b8-2e8c6b37613e (af1123f4-626a-46f8-a46e-4008ec56e922) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.812 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.812 INFO bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 WaitActive WaitActive WaitActive
Sep 22 23:21:37.812 WARN [1] new RM replaced this: None
Sep 22 23:21:37.812 INFO [1] Starts reconcile loop
Sep 22 23:21:37.812 INFO [2] downstairs client at 127.0.0.1:55757 has UUID e05c70a3-48dd-4fb2-827c-e28061a0ccac
Sep 22 23:21:37.812 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e05c70a3-48dd-4fb2-827c-e28061a0ccac, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.812 INFO bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 WaitActive WaitActive WaitActive
Sep 22 23:21:37.812 INFO [2] 8a9244fd-a219-4407-95b8-2e8c6b37613e (af1123f4-626a-46f8-a46e-4008ec56e922) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.812 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.812 WARN [2] new RM replaced this: None
Sep 22 23:21:37.812 INFO [1] downstairs client at 127.0.0.1:36989 has UUID e251ad5f-6614-4ef0-b821-00d45953fcfc
Sep 22 23:21:37.812 INFO [2] Starts reconcile loop
Sep 22 23:21:37.812 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e251ad5f-6614-4ef0-b821-00d45953fcfc, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.812 INFO bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 WaitActive WaitActive WaitActive
Sep 22 23:21:37.812 INFO [0] 127.0.0.1:64153 task reports connection:true
Sep 22 23:21:37.812 INFO 8a9244fd-a219-4407-95b8-2e8c6b37613e WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.812 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.812 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.812 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.812 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.812 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.812 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.812 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.812 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.812 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.812 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.812 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.812 INFO Max found gen is 1
Sep 22 23:21:37.812 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.812 INFO Next flush: 1
Sep 22 23:21:37.812 INFO All extents match
Sep 22 23:21:37.812 INFO No downstairs repair required
Sep 22 23:21:37.812 INFO No initial repair work was required
Sep 22 23:21:37.812 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.812 INFO 8a9244fd-a219-4407-95b8-2e8c6b37613e is now active with session: af1123f4-626a-46f8-a46e-4008ec56e922
Sep 22 23:21:37.812 INFO 8a9244fd-a219-4407-95b8-2e8c6b37613e Set Active after no repair
Sep 22 23:21:37.812 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.812 INFO Set check for repair
Sep 22 23:21:37.812 INFO [1] 127.0.0.1:59973 task reports connection:true
Sep 22 23:21:37.812 INFO 8a9244fd-a219-4407-95b8-2e8c6b37613e Active Active Active
Sep 22 23:21:37.812 INFO Set check for repair
Sep 22 23:21:37.812 INFO [2] 127.0.0.1:46588 task reports connection:true
Sep 22 23:21:37.812 INFO 8a9244fd-a219-4407-95b8-2e8c6b37613e Active Active Active
Sep 22 23:21:37.812 INFO Set check for repair
Sep 22 23:21:37.813 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.813 INFO [0] received reconcile message
Sep 22 23:21:37.813 INFO [0] All repairs completed, exit
Sep 22 23:21:37.813 INFO [0] Starts cmd_loop
Sep 22 23:21:37.813 INFO [1] received reconcile message
Sep 22 23:21:37.813 INFO [1] All repairs completed, exit
Sep 22 23:21:37.813 INFO [1] Starts cmd_loop
Sep 22 23:21:37.813 INFO [2] received reconcile message
Sep 22 23:21:37.813 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.813 INFO [2] All repairs completed, exit
Sep 22 23:21:37.813 INFO [2] Starts cmd_loop
Sep 22 23:21:37.813 DEBG Read      :1000 deps:[] res:true
The guest has finished waiting for activation
Sep 22 23:21:37.813 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.813 INFO UUID: 9278686c-4d89-49de-9739-9711102f9bee
Sep 22 23:21:37.813 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.813 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.813 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.813 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.813 INFO Using address: 127.0.0.1:58397, task: main
Sep 22 23:21:37.813 DEBG IO Read  1000 has deps []
test test::integration_test_region ... ok
Sep 22 23:21:37.814 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.814 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.814 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.814 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:64974, task: repair
Sep 22 23:21:37.814 INFO [0] bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 (82d82a34-37ca-41ad-83f4-a1bf54c18aff) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.814 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.814 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:64974, task: repair
Sep 22 23:21:37.814 WARN [0] new RM replaced this: None
Sep 22 23:21:37.814 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.814 INFO listening, local_addr: 127.0.0.1:64974, task: repair
Sep 22 23:21:37.814 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.814 INFO [0] Starts reconcile loop
Sep 22 23:21:37.814 INFO Created new region file "/tmp/downstairs-bxfe8IV2/region.json"
Sep 22 23:21:37.814 INFO [2] bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 (82d82a34-37ca-41ad-83f4-a1bf54c18aff) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.814 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.814 WARN [2] new RM replaced this: None
Sep 22 23:21:37.814 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.814 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:64974, task: repair
Sep 22 23:21:37.814 INFO [2] Starts reconcile loop
Sep 22 23:21:37.814 DEBG up_ds_listen was notified
Sep 22 23:21:37.814 INFO Using repair address: 127.0.0.1:64974, task: main
Sep 22 23:21:37.814 DEBG up_ds_listen process 1000
Sep 22 23:21:37.814 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.814 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.814 DEBG IO Write 1003 has deps []
Sep 22 23:21:37.814 INFO [1] bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 (82d82a34-37ca-41ad-83f4-a1bf54c18aff) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.814 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.814 WARN [1] new RM replaced this: None
Sep 22 23:21:37.814 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.814 INFO [1] Starts reconcile loop
Sep 22 23:21:37.814 INFO [0] 127.0.0.1:33453 task reports connection:true
Sep 22 23:21:37.814 INFO bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.814 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.814 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.814 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.814 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.814 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.814 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.814 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.814 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.815 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.815 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.815 INFO Max found gen is 1
Sep 22 23:21:37.815 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.815 INFO Next flush: 1
Sep 22 23:21:37.815 INFO All extents match
Sep 22 23:21:37.815 INFO No downstairs repair required
Sep 22 23:21:37.815 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.815 INFO No initial repair work was required
Sep 22 23:21:37.815 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.815 INFO bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 is now active with session: 82d82a34-37ca-41ad-83f4-a1bf54c18aff
Sep 22 23:21:37.815 INFO bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 Set Active after no repair
Sep 22 23:21:37.815 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.815 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.815 INFO Set check for repair
Sep 22 23:21:37.815 INFO [2] 127.0.0.1:55757 task reports connection:true
Sep 22 23:21:37.815 INFO bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 Active Active Active
Sep 22 23:21:37.815 INFO Set check for repair
Sep 22 23:21:37.815 INFO [1] 127.0.0.1:36989 task reports connection:true
Sep 22 23:21:37.815 INFO bc449dbb-4e8d-49d4-9ec4-c4ddf333de98 Active Active Active
Sep 22 23:21:37.815 INFO Set check for repair
Sep 22 23:21:37.815 INFO [0] received reconcile message
Sep 22 23:21:37.815 INFO [0] All repairs completed, exit
Sep 22 23:21:37.815 INFO [0] Starts cmd_loop
Sep 22 23:21:37.815 INFO Upstairs starts
Sep 22 23:21:37.815 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.815 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.815 INFO Crucible stats registered with UUID: 1ba80387-235e-4d57-bd6a-ddde20bc88f5
Sep 22 23:21:37.815 INFO [1] received reconcile message
Sep 22 23:21:37.815 INFO [1] All repairs completed, exit
Sep 22 23:21:37.815 INFO Crucible 1ba80387-235e-4d57-bd6a-ddde20bc88f5 has session id: 930575d8-984d-4b49-ae80-e9a9c3c00aea
Sep 22 23:21:37.815 INFO [1] Starts cmd_loop
Sep 22 23:21:37.815 INFO [2] received reconcile message
Sep 22 23:21:37.815 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.816 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.816 INFO [2] All repairs completed, exit
Sep 22 23:21:37.816 INFO [2] Starts cmd_loop
Sep 22 23:21:37.816 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.816 INFO listening on 127.0.0.1:0, task: main
The guest has finished waiting for activation
Sep 22 23:21:37.816 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.816 INFO [0] connecting to 127.0.0.1:38477, looper: 0
Sep 22 23:21:37.816 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.816 INFO [1] connecting to 127.0.0.1:39308, looper: 1
Sep 22 23:21:37.816 DEBG up_ds_listen was notified
Sep 22 23:21:37.816 DEBG up_ds_listen process 1000
Sep 22 23:21:37.816 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.816 INFO [2] connecting to 127.0.0.1:58397, looper: 2
Sep 22 23:21:37.816 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.816 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.816 INFO Flush timeout: 0.5
Sep 22 23:21:37.816 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.816 DEBG IO Write 1000 has deps []
Sep 22 23:21:37.816 INFO accepted connection from 127.0.0.1:51609, task: main
Sep 22 23:21:37.816 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:37.816 DEBG up_ds_listen was notified
Sep 22 23:21:37.816 DEBG up_ds_listen process 1000
Sep 22 23:21:37.816 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.816 INFO accepted connection from 127.0.0.1:50744, task: main
Sep 22 23:21:37.816 DEBG up_ds_listen was notified
Sep 22 23:21:37.817 DEBG up_ds_listen process 1001
Sep 22 23:21:37.817 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.817 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.817 INFO [0] 1ba80387-235e-4d57-bd6a-ddde20bc88f5 looper connected, looper: 0
Sep 22 23:21:37.817 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.817 INFO [0] Proc runs for 127.0.0.1:38477 in state New
Sep 22 23:21:37.817 INFO accepted connection from 127.0.0.1:64280, task: main
Sep 22 23:21:37.817 INFO [2] 1ba80387-235e-4d57-bd6a-ddde20bc88f5 looper connected, looper: 2
Sep 22 23:21:37.817 INFO [2] Proc runs for 127.0.0.1:58397 in state New
Sep 22 23:21:37.817 INFO [1] 1ba80387-235e-4d57-bd6a-ddde20bc88f5 looper connected, looper: 1
Sep 22 23:21:37.817 INFO [1] Proc runs for 127.0.0.1:39308 in state New
Sep 22 23:21:37.817 DEBG up_ds_listen was notified
Sep 22 23:21:37.817 DEBG up_ds_listen process 1003
Sep 22 23:21:37.817 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:37.817 DEBG IO Read  1001 has deps []
Sep 22 23:21:37.817 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.817 INFO Scrub check for 75f46847-6259-4674-b0dd-3bfaab313ce5
Sep 22 23:21:37.817 INFO Scrub for 75f46847-6259-4674-b0dd-3bfaab313ce5 not required
Sep 22 23:21:37.817 INFO Connection request from 1ba80387-235e-4d57-bd6a-ddde20bc88f5 with version 4, task: proc
Sep 22 23:21:37.817 DEBG IO Read  1004 has deps [JobId(1003)]
Sep 22 23:21:37.817 INFO upstairs UpstairsConnection { upstairs_id: 1ba80387-235e-4d57-bd6a-ddde20bc88f5, session_id: 225d651b-211c-4da3-aced-2077e6a64214, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.817 INFO Connection request from 1ba80387-235e-4d57-bd6a-ddde20bc88f5 with version 4, task: proc
Sep 22 23:21:37.817 INFO upstairs UpstairsConnection { upstairs_id: 1ba80387-235e-4d57-bd6a-ddde20bc88f5, session_id: 225d651b-211c-4da3-aced-2077e6a64214, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.818 INFO Connection request from 1ba80387-235e-4d57-bd6a-ddde20bc88f5 with version 4, task: proc
Sep 22 23:21:37.818 INFO upstairs UpstairsConnection { upstairs_id: 1ba80387-235e-4d57-bd6a-ddde20bc88f5, session_id: 225d651b-211c-4da3-aced-2077e6a64214, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.818 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.818 INFO Opened existing region file "/tmp/downstairs-bxfe8IV2/region.json"
Sep 22 23:21:37.818 INFO Database read version 1
Sep 22 23:21:37.818 INFO Database write version 1
The guest has requested activation
Sep 22 23:21:37.818 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:37.818 INFO [0] 1ba80387-235e-4d57-bd6a-ddde20bc88f5 (225d651b-211c-4da3-aced-2077e6a64214) New New New ds_transition to WaitActive
Sep 22 23:21:37.818 DEBG Read      :1001 deps:[] res:true
Sep 22 23:21:37.818 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.818 DEBG Read      :1004 deps:[JobId(1003)] res:true
Sep 22 23:21:37.818 INFO [2] 1ba80387-235e-4d57-bd6a-ddde20bc88f5 (225d651b-211c-4da3-aced-2077e6a64214) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.818 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.818 INFO [1] 1ba80387-235e-4d57-bd6a-ddde20bc88f5 (225d651b-211c-4da3-aced-2077e6a64214) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:21:37.818 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.818 DEBG Read      :1001 deps:[] res:true
Sep 22 23:21:37.818 INFO 1ba80387-235e-4d57-bd6a-ddde20bc88f5 active request set
Sep 22 23:21:37.818 INFO [0] received activate with gen 1
Sep 22 23:21:37.818 DEBG Read      :1004 deps:[JobId(1003)] res:true
Sep 22 23:21:37.818 INFO [0] client got ds_active_rx, promote! session 225d651b-211c-4da3-aced-2077e6a64214
Sep 22 23:21:37.818 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:37.818 INFO [1] received activate with gen 1
Sep 22 23:21:37.818 DEBG Read      :1001 deps:[] res:true
Sep 22 23:21:37.818 INFO [1] client got ds_active_rx, promote! session 225d651b-211c-4da3-aced-2077e6a64214
Sep 22 23:21:37.818 INFO [2] received activate with gen 1
Sep 22 23:21:37.818 INFO [2] client got ds_active_rx, promote! session 225d651b-211c-4da3-aced-2077e6a64214
Sep 22 23:21:37.819 DEBG Read      :1004 deps:[JobId(1003)] res:true
Sep 22 23:21:37.819 INFO UpstairsConnection { upstairs_id: 1ba80387-235e-4d57-bd6a-ddde20bc88f5, session_id: 225d651b-211c-4da3-aced-2077e6a64214, gen: 1 } is now active (read-write)
Sep 22 23:21:37.819 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:37.819 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.819 INFO UpstairsConnection { upstairs_id: 1ba80387-235e-4d57-bd6a-ddde20bc88f5, session_id: 225d651b-211c-4da3-aced-2077e6a64214, gen: 1 } is now active (read-write)
Sep 22 23:21:37.819 INFO UpstairsConnection { upstairs_id: 1ba80387-235e-4d57-bd6a-ddde20bc88f5, session_id: 225d651b-211c-4da3-aced-2077e6a64214, gen: 1 } is now active (read-write)
Sep 22 23:21:37.819 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:37.819 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.819 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:37.819 DEBG IO Write 1001 has deps []
Sep 22 23:21:37.819 INFO [0] downstairs client at 127.0.0.1:38477 has UUID 225348bb-ea71-4678-9320-8b37dbfcd881
Sep 22 23:21:37.819 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 225348bb-ea71-4678-9320-8b37dbfcd881, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.819 INFO 1ba80387-235e-4d57-bd6a-ddde20bc88f5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.819 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:37.819 INFO [2] downstairs client at 127.0.0.1:58397 has UUID 9278686c-4d89-49de-9739-9711102f9bee
Sep 22 23:21:37.819 DEBG up_ds_listen was notified
Sep 22 23:21:37.819 DEBG up_ds_listen was notified
Sep 22 23:21:37.820 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9278686c-4d89-49de-9739-9711102f9bee, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.820 DEBG up_ds_listen process 1001
Sep 22 23:21:37.820 DEBG up_ds_listen process 1001
Sep 22 23:21:37.820 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.820 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.820 INFO 1ba80387-235e-4d57-bd6a-ddde20bc88f5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.820 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.820 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.820 INFO [1] downstairs client at 127.0.0.1:39308 has UUID 2a37b56b-50b2-4d7c-9cf7-bf9557fd6093
Sep 22 23:21:37.820 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2a37b56b-50b2-4d7c-9cf7-bf9557fd6093, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.820 INFO 1ba80387-235e-4d57-bd6a-ddde20bc88f5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.820 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.820 INFO Scrub check for 507208f9-64c5-4508-b304-2a8e206b5aec
Sep 22 23:21:37.820 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:21:37.820 INFO Scrub for 507208f9-64c5-4508-b304-2a8e206b5aec begins
Sep 22 23:21:37.820 INFO Scrub with total_size:2560 block_size:512
Sep 22 23:21:37.820 INFO Scrub check for 09bdd5d2-3b23-4357-8a7e-90e2cd35ea46
Sep 22 23:21:37.820 INFO Scrubs from block 0 to 5 in (256) 131072 size IOs pm:0
Sep 22 23:21:37.820 INFO Adjust block_count to 5 at offset 0
Sep 22 23:21:37.820 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.820 INFO Scrub for 09bdd5d2-3b23-4357-8a7e-90e2cd35ea46 begins
Sep 22 23:21:37.820 INFO Scrub with total_size:2560 block_size:512
Sep 22 23:21:37.820 INFO Scrubs from block 0 to 5 in (256) 131072 size IOs pm:0
Sep 22 23:21:37.820 INFO Adjust block_count to 5 at offset 0
Sep 22 23:21:37.820 INFO UUID: f5ea863d-974b-4712-8f07-7666c185979b
Sep 22 23:21:37.820 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.820 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.820 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.820 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.820 INFO Using address: 127.0.0.1:65137, task: main
Sep 22 23:21:37.820 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.821 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.821 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.821 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53949, task: repair
Sep 22 23:21:37.821 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.821 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53949, task: repair
Sep 22 23:21:37.821 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.821 INFO listening, local_addr: 127.0.0.1:53949, task: repair
Sep 22 23:21:37.821 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.821 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53949, task: repair
Sep 22 23:21:37.821 INFO Using repair address: 127.0.0.1:53949, task: main
Sep 22 23:21:37.821 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.821 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.821 INFO [0] 1ba80387-235e-4d57-bd6a-ddde20bc88f5 (225d651b-211c-4da3-aced-2077e6a64214) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.821 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.821 WARN [0] new RM replaced this: None
Sep 22 23:21:37.821 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:37.821 INFO [0] Starts reconcile loop
Sep 22 23:21:37.821 INFO [2] 1ba80387-235e-4d57-bd6a-ddde20bc88f5 (225d651b-211c-4da3-aced-2077e6a64214) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.821 DEBG [0] Read AckReady 1004, : downstairs
Sep 22 23:21:37.821 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.821 WARN [2] new RM replaced this: None
Sep 22 23:21:37.821 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.821 INFO [2] Starts reconcile loop
Sep 22 23:21:37.821 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.822 INFO [1] 1ba80387-235e-4d57-bd6a-ddde20bc88f5 (225d651b-211c-4da3-aced-2077e6a64214) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.822 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.822 WARN [1] new RM replaced this: None
Sep 22 23:21:37.822 INFO Created new region file "/tmp/downstairs-RQXy3Y00/region.json"
Sep 22 23:21:37.822 INFO [1] Starts reconcile loop
Sep 22 23:21:37.822 INFO [0] 127.0.0.1:38477 task reports connection:true
Sep 22 23:21:37.822 INFO 1ba80387-235e-4d57-bd6a-ddde20bc88f5 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.822 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.822 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.822 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.822 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:37.822 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.822 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.822 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.822 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.822 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.822 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.822 INFO Max found gen is 1
Sep 22 23:21:37.822 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.822 INFO Next flush: 1
Sep 22 23:21:37.822 DEBG IO Write 1002 has deps [JobId(1000)]
Sep 22 23:21:37.822 INFO All extents match
Sep 22 23:21:37.822 INFO No downstairs repair required
Sep 22 23:21:37.822 INFO No initial repair work was required
Sep 22 23:21:37.822 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.822 INFO 1ba80387-235e-4d57-bd6a-ddde20bc88f5 is now active with session: 225d651b-211c-4da3-aced-2077e6a64214
Sep 22 23:21:37.822 INFO 1ba80387-235e-4d57-bd6a-ddde20bc88f5 Set Active after no repair
Sep 22 23:21:37.822 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.822 INFO Set check for repair
Sep 22 23:21:37.822 INFO [2] 127.0.0.1:58397 task reports connection:true
Sep 22 23:21:37.822 INFO 1ba80387-235e-4d57-bd6a-ddde20bc88f5 Active Active Active
Sep 22 23:21:37.822 INFO Set check for repair
Sep 22 23:21:37.822 INFO [1] 127.0.0.1:39308 task reports connection:true
Sep 22 23:21:37.822 INFO 1ba80387-235e-4d57-bd6a-ddde20bc88f5 Active Active Active
Sep 22 23:21:37.822 INFO Set check for repair
Sep 22 23:21:37.822 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:37.822 INFO [0] received reconcile message
Sep 22 23:21:37.822 INFO [0] All repairs completed, exit
Sep 22 23:21:37.822 INFO [0] Starts cmd_loop
Sep 22 23:21:37.822 INFO [1] received reconcile message
Sep 22 23:21:37.823 INFO [1] All repairs completed, exit
Sep 22 23:21:37.823 INFO [1] Starts cmd_loop
Sep 22 23:21:37.823 INFO [2] received reconcile message
Sep 22 23:21:37.823 INFO [2] All repairs completed, exit
Sep 22 23:21:37.823 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:37.823 DEBG [1] Read already AckReady 1004, : downstairs
Sep 22 23:21:37.823 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.824 DEBG IO Write 1002 has deps [JobId(1000)]
Sep 22 23:21:37.824 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:21:37.824 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.825 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.825 DEBG [2] Read already AckReady 1004, : downstairs
Sep 22 23:21:37.825 DEBG up_ds_listen was notified
Sep 22 23:21:37.825 DEBG up_ds_listen process 1004
Sep 22 23:21:37.825 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.825 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:21:37.825 INFO Opened existing region file "/tmp/downstairs-RQXy3Y00/region.json"
Sep 22 23:21:37.825 INFO Database read version 1
Sep 22 23:21:37.825 INFO Database write version 1
Sep 22 23:21:37.825 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.825 DEBG up_ds_listen was notified
Sep 22 23:21:37.825 DEBG up_ds_listen process 1002
Sep 22 23:21:37.825 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.825 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.825 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.826 INFO Scrub at offset 5/5 sp:5
Sep 22 23:21:37.826 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.826 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.827 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.827 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.827 INFO Scrub 507208f9-64c5-4508-b304-2a8e206b5aec done in 0 seconds. Retries:0 scrub_size:2560 size:5 pause_milli:0
Sep 22 23:21:37.827 DEBG up_ds_listen was notified
Sep 22 23:21:37.827 DEBG up_ds_listen process 1002
Sep 22 23:21:37.827 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.827 DEBG IO Flush 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:21:37.827 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.827 DEBG up_ds_listen was notified
Sep 22 23:21:37.827 DEBG up_ds_listen process 1000
Sep 22 23:21:37.827 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.827 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.827 INFO UUID: a1447c8e-cf80-4bea-b730-010907ef0c7f
Sep 22 23:21:37.827 DEBG up_ds_listen was notified
Sep 22 23:21:37.827 DEBG up_ds_listen process 1002
Sep 22 23:21:37.827 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.828 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.828 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.828 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.828 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.828 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.828 INFO Using address: 127.0.0.1:44628, task: main
Sep 22 23:21:37.828 INFO Scrub at offset 5/5 sp:5
Sep 22 23:21:37.828 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.828 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42546, task: repair
Sep 22 23:21:37.828 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42546, task: repair
Sep 22 23:21:37.828 INFO listening, local_addr: 127.0.0.1:42546, task: repair
Sep 22 23:21:37.828 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42546, task: repair
Sep 22 23:21:37.828 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.828 INFO Using repair address: 127.0.0.1:42546, task: main
Sep 22 23:21:37.828 INFO No SSL acceptor configured, task: main
test test::integration_test_scrub ... ok
Sep 22 23:21:37.828 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.829 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.829 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.829 INFO Created new region file "/tmp/downstairs-agB7RjDz/region.json"
Sep 22 23:21:37.829 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.829 INFO Created new region file "/tmp/downstairs-FzhHxHPv/region.json"
Sep 22 23:21:37.829 DEBG up_ds_listen was notified
Sep 22 23:21:37.829 DEBG up_ds_listen process 1003
Sep 22 23:21:37.829 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:37.829 INFO Scrub 09bdd5d2-3b23-4357-8a7e-90e2cd35ea46 done in 0 seconds. Retries:0 scrub_size:2560 size:5 pause_milli:0
Sep 22 23:21:37.829 DEBG [rc] retire 1003 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003)], : downstairs
Sep 22 23:21:37.829 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.829 DEBG IO Flush 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
test test::integration_test_scrub_no_rop ... ok
Sep 22 23:21:37.832 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.832 INFO Created new region file "/tmp/downstairs-UTqRCVgq/region.json"
Sep 22 23:21:37.832 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:37.832 DEBG IO Write 1004 has deps []
Sep 22 23:21:37.832 DEBG up_ds_listen was notified
Sep 22 23:21:37.832 DEBG up_ds_listen process 1001
Sep 22 23:21:37.832 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.832 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.832 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.833 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.833 INFO Opened existing region file "/tmp/downstairs-FzhHxHPv/region.json"
Sep 22 23:21:37.833 INFO Database read version 1
Sep 22 23:21:37.833 INFO Database write version 1
Sep 22 23:21:37.833 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.833 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.833 DEBG up_ds_listen was notified
Sep 22 23:21:37.833 DEBG up_ds_listen process 1003
Sep 22 23:21:37.833 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:37.833 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.833 INFO Opened existing region file "/tmp/downstairs-agB7RjDz/region.json"
Sep 22 23:21:37.833 INFO Database read version 1
Sep 22 23:21:37.833 DEBG [rc] retire 1003 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003)], : downstairs
Sep 22 23:21:37.833 INFO Database write version 1
Sep 22 23:21:37.833 INFO Scrub check for 854c46f9-33d9-4450-b666-b1c944841aff
Sep 22 23:21:37.833 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.833 INFO Scrub for 854c46f9-33d9-4450-b666-b1c944841aff begins
Sep 22 23:21:37.833 INFO Scrub with total_size:5120 block_size:512
Sep 22 23:21:37.833 INFO Scrubs from block 0 to 10 in (256) 131072 size IOs pm:0
Sep 22 23:21:37.833 INFO Adjust block_count to 10 at offset 0
Sep 22 23:21:37.833 DEBG IO Read  1004 has deps []
Sep 22 23:21:37.834 DEBG Read      :1004 deps:[] res:true
Sep 22 23:21:37.835 INFO UUID: dc8a7c43-ebc6-4a48-b659-c51804dc92bb
Sep 22 23:21:37.835 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.835 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.835 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.835 INFO Using address: 127.0.0.1:46928, task: main
Sep 22 23:21:37.835 DEBG Read      :1004 deps:[] res:true
Sep 22 23:21:37.835 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.835 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:54347, task: repair
Sep 22 23:21:37.835 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:54347, task: repair
Sep 22 23:21:37.835 INFO listening, local_addr: 127.0.0.1:54347, task: repair
Sep 22 23:21:37.835 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.835 INFO Opened existing region file "/tmp/downstairs-UTqRCVgq/region.json"
Sep 22 23:21:37.835 INFO Database read version 1
Sep 22 23:21:37.835 INFO Database write version 1
Sep 22 23:21:37.836 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:54347, task: repair
Sep 22 23:21:37.836 DEBG Read      :1004 deps:[] res:true
Sep 22 23:21:37.836 INFO UUID: 1b0c673f-012e-4936-8109-087022151153
Sep 22 23:21:37.836 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.836 INFO Using repair address: 127.0.0.1:54347, task: main
Sep 22 23:21:37.836 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.836 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.836 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.836 INFO Using address: 127.0.0.1:38317, task: main
Sep 22 23:21:37.836 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.836 INFO Created new region file "/tmp/downstairs-slwJmBq9/region.json"
Sep 22 23:21:37.836 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.836 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60003, task: repair
Sep 22 23:21:37.836 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60003, task: repair
Sep 22 23:21:37.836 INFO listening, local_addr: 127.0.0.1:60003, task: repair
Sep 22 23:21:37.836 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60003, task: repair
Sep 22 23:21:37.836 INFO Using repair address: 127.0.0.1:60003, task: main
Sep 22 23:21:37.836 DEBG up_ds_listen was notified
Sep 22 23:21:37.836 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.837 DEBG up_ds_listen process 1004
Sep 22 23:21:37.837 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:21:37.837 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.837 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.837 INFO UUID: dc9055ff-ffac-4e79-bd57-721a805ccf83
Sep 22 23:21:37.837 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.837 DEBG IO Read  1005 has deps [JobId(1004)]
Sep 22 23:21:37.837 INFO Upstairs starts
Sep 22 23:21:37.837 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.837 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.837 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.837 INFO Using address: 127.0.0.1:34995, task: main
Sep 22 23:21:37.837 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.837 INFO Crucible stats registered with UUID: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5
Sep 22 23:21:37.837 INFO Crucible 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 has session id: 44811dc3-2625-4058-b159-bbab926f9643
Sep 22 23:21:37.837 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.837 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.837 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:64816, task: repair
Sep 22 23:21:37.837 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.837 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:64816, task: repair
Sep 22 23:21:37.837 INFO listening, local_addr: 127.0.0.1:64816, task: repair
Sep 22 23:21:37.837 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.837 INFO [0] connecting to 127.0.0.1:65137, looper: 0
Sep 22 23:21:37.837 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:64816, task: repair
Sep 22 23:21:37.838 INFO Using repair address: 127.0.0.1:64816, task: main
Sep 22 23:21:37.838 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.838 INFO [1] connecting to 127.0.0.1:44628, looper: 1
Sep 22 23:21:37.838 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.838 INFO [2] connecting to 127.0.0.1:38317, looper: 2
Sep 22 23:21:37.838 DEBG Read      :1005 deps:[JobId(1004)] res:true
Sep 22 23:21:37.838 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.838 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.838 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.838 INFO Flush timeout: 0.5
Sep 22 23:21:37.838 INFO Created new region file "/tmp/downstairs-NmE9CZeD/region.json"
Sep 22 23:21:37.838 INFO accepted connection from 127.0.0.1:43896, task: main
Sep 22 23:21:37.838 DEBG Read      :1005 deps:[JobId(1004)] res:true
Sep 22 23:21:37.838 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 looper connected, looper: 1
Sep 22 23:21:37.838 INFO [1] Proc runs for 127.0.0.1:44628 in state New
Sep 22 23:21:37.838 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 looper connected, looper: 2
Sep 22 23:21:37.838 INFO [2] Proc runs for 127.0.0.1:38317 in state New
Sep 22 23:21:37.838 INFO accepted connection from 127.0.0.1:47893, task: main
Sep 22 23:21:37.839 INFO accepted connection from 127.0.0.1:48011, task: main
Sep 22 23:21:37.839 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.839 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 looper connected, looper: 0
Sep 22 23:21:37.839 DEBG Read      :1005 deps:[JobId(1004)] res:true
Sep 22 23:21:37.839 INFO [0] Proc runs for 127.0.0.1:65137 in state New
Sep 22 23:21:37.839 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.839 INFO Opened existing region file "/tmp/downstairs-slwJmBq9/region.json"
Sep 22 23:21:37.839 INFO Database read version 1
Sep 22 23:21:37.839 INFO Database write version 1
Sep 22 23:21:37.839 DEBG [0] Read AckReady 1004, : downstairs
Sep 22 23:21:37.839 INFO Connection request from 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 with version 4, task: proc
Sep 22 23:21:37.839 INFO upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.839 INFO Connection request from 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 with version 4, task: proc
Sep 22 23:21:37.839 INFO upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.839 INFO Connection request from 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 with version 4, task: proc
Sep 22 23:21:37.839 INFO upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:21:37.839 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (c9717b37-7d6a-40f7-9030-e7e57914f334) New New New ds_transition to WaitActive
Sep 22 23:21:37.839 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.840 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (c9717b37-7d6a-40f7-9030-e7e57914f334) New WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.840 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.840 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (c9717b37-7d6a-40f7-9030-e7e57914f334) New WaitActive WaitActive ds_transition to WaitActive
Sep 22 23:21:37.840 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.840 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 active request set
Sep 22 23:21:37.840 INFO [0] received activate with gen 1
Sep 22 23:21:37.840 INFO [0] client got ds_active_rx, promote! session c9717b37-7d6a-40f7-9030-e7e57914f334
Sep 22 23:21:37.840 INFO [1] received activate with gen 1
Sep 22 23:21:37.840 INFO [1] client got ds_active_rx, promote! session c9717b37-7d6a-40f7-9030-e7e57914f334
Sep 22 23:21:37.840 INFO [2] received activate with gen 1
Sep 22 23:21:37.840 INFO [2] client got ds_active_rx, promote! session c9717b37-7d6a-40f7-9030-e7e57914f334
Sep 22 23:21:37.840 INFO UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } is now active (read-write)
Sep 22 23:21:37.840 INFO UUID: eb9758aa-5126-4e8c-b409-22a2bcacbe1b
Sep 22 23:21:37.840 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.840 INFO UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } is now active (read-write)
Sep 22 23:21:37.840 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.840 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.840 INFO Using address: 127.0.0.1:43209, task: main
Sep 22 23:21:37.840 INFO UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } is now active (read-write)
Sep 22 23:21:37.840 DEBG [2] Read already AckReady 1004, : downstairs
Sep 22 23:21:37.841 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.841 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51701, task: repair
Sep 22 23:21:37.841 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51701, task: repair
Sep 22 23:21:37.841 INFO listening, local_addr: 127.0.0.1:51701, task: repair
Sep 22 23:21:37.841 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51701, task: repair
Sep 22 23:21:37.841 INFO Using repair address: 127.0.0.1:51701, task: main
Sep 22 23:21:37.841 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.841 INFO [1] downstairs client at 127.0.0.1:44628 has UUID a1447c8e-cf80-4bea-b730-010907ef0c7f
Sep 22 23:21:37.841 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a1447c8e-cf80-4bea-b730-010907ef0c7f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.841 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.841 INFO [2] downstairs client at 127.0.0.1:38317 has UUID 1b0c673f-012e-4936-8109-087022151153
Sep 22 23:21:37.841 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1b0c673f-012e-4936-8109-087022151153, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.841 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.841 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.841 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.841 INFO Opened existing region file "/tmp/downstairs-NmE9CZeD/region.json"
Sep 22 23:21:37.841 INFO Created new region file "/tmp/downstairs-usAyrxwT/region.json"
Sep 22 23:21:37.841 INFO Database read version 1
Sep 22 23:21:37.841 INFO Database write version 1
Sep 22 23:21:37.841 INFO [0] downstairs client at 127.0.0.1:65137 has UUID f5ea863d-974b-4712-8f07-7666c185979b
Sep 22 23:21:37.841 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f5ea863d-974b-4712-8f07-7666c185979b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.841 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.842 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.842 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.842 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:37.842 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.842 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.842 DEBG [1] Read already AckReady 1004, : downstairs
Sep 22 23:21:37.842 DEBG up_ds_listen was notified
Sep 22 23:21:37.842 DEBG up_ds_listen process 1004
Sep 22 23:21:37.842 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:21:37.842 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.843 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.843 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (c9717b37-7d6a-40f7-9030-e7e57914f334) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.843 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.843 WARN [1] new RM replaced this: None
Sep 22 23:21:37.843 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.843 INFO [1] Starts reconcile loop
Sep 22 23:21:37.843 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (c9717b37-7d6a-40f7-9030-e7e57914f334) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.843 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.843 WARN [2] new RM replaced this: None
Sep 22 23:21:37.843 INFO [2] Starts reconcile loop
Sep 22 23:21:37.843 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (c9717b37-7d6a-40f7-9030-e7e57914f334) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.843 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.843 WARN [0] new RM replaced this: None
Sep 22 23:21:37.843 DEBG [0] Read AckReady 1005, : downstairs
Sep 22 23:21:37.843 INFO [0] Starts reconcile loop
Sep 22 23:21:37.843 INFO [1] 127.0.0.1:44628 task reports connection:true
Sep 22 23:21:37.843 INFO UUID: 2e1bc31f-575d-410d-bb06-222733a39c1e
Sep 22 23:21:37.843 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.843 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.843 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.844 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.844 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.844 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.844 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.844 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.844 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.844 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.844 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.844 INFO Using address: 127.0.0.1:45660, task: main
Sep 22 23:21:37.844 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.844 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.844 INFO Max found gen is 1
Sep 22 23:21:37.844 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.844 INFO Next flush: 1
Sep 22 23:21:37.844 INFO All extents match
Sep 22 23:21:37.844 INFO No downstairs repair required
Sep 22 23:21:37.844 INFO No initial repair work was required
Sep 22 23:21:37.844 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.844 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 is now active with session: c9717b37-7d6a-40f7-9030-e7e57914f334
Sep 22 23:21:37.844 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Set Active after no repair
Sep 22 23:21:37.844 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.844 INFO Set check for repair
Sep 22 23:21:37.844 INFO [2] 127.0.0.1:38317 task reports connection:true
Sep 22 23:21:37.844 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Active Active Active
Sep 22 23:21:37.844 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.844 INFO Set check for repair
Sep 22 23:21:37.844 INFO [0] 127.0.0.1:65137 task reports connection:true
Sep 22 23:21:37.844 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Active Active Active
Sep 22 23:21:37.844 INFO Set check for repair
Sep 22 23:21:37.844 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33168, task: repair
Sep 22 23:21:37.844 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33168, task: repair
Sep 22 23:21:37.844 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.844 INFO Opened existing region file "/tmp/downstairs-usAyrxwT/region.json"
Sep 22 23:21:37.844 INFO Database read version 1
Sep 22 23:21:37.844 INFO [0] received reconcile message
Sep 22 23:21:37.844 INFO Database write version 1
Sep 22 23:21:37.844 INFO listening, local_addr: 127.0.0.1:33168, task: repair
Sep 22 23:21:37.844 INFO [0] All repairs completed, exit
Sep 22 23:21:37.844 INFO [0] Starts cmd_loop
Sep 22 23:21:37.844 INFO [1] received reconcile message
Sep 22 23:21:37.844 INFO [1] All repairs completed, exit
Sep 22 23:21:37.844 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33168, task: repair
Sep 22 23:21:37.844 INFO [1] Starts cmd_loop
Sep 22 23:21:37.844 INFO [2] received reconcile message
Sep 22 23:21:37.844 INFO [2] All repairs completed, exit
Sep 22 23:21:37.844 INFO Using repair address: 127.0.0.1:33168, task: main
Sep 22 23:21:37.844 INFO [2] Starts cmd_loop
Sep 22 23:21:37.844 INFO No SSL acceptor configured, task: main
The guest has finished waiting for activation
Sep 22 23:21:37.845 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.845 INFO Created new region file "/tmp/downstairs-w5PkmiSc/region.json"
Sep 22 23:21:37.846 DEBG [1] Read already AckReady 1005, : downstairs
Sep 22 23:21:37.846 DEBG up_ds_listen was notified
Sep 22 23:21:37.846 DEBG up_ds_listen process 1002
Sep 22 23:21:37.846 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.846 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.846 INFO Scrub at offset 10/10 sp:10
Sep 22 23:21:37.847 INFO UUID: 8472a597-92db-4cea-a2e5-658ef1a3d615
Sep 22 23:21:37.847 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.847 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.847 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.847 INFO Using address: 127.0.0.1:44686, task: main
Sep 22 23:21:37.848 INFO Scrub 854c46f9-33d9-4450-b666-b1c944841aff done in 0 seconds. Retries:0 scrub_size:5120 size:10 pause_milli:0
Sep 22 23:21:37.848 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.848 DEBG IO Flush 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:21:37.848 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51964, task: repair
Sep 22 23:21:37.848 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51964, task: repair
Sep 22 23:21:37.848 DEBG IO Write 1000 has deps []
Sep 22 23:21:37.848 INFO listening, local_addr: 127.0.0.1:51964, task: repair
test test::integration_test_scrub_short_sparse ... Sep 22 23:21:37.848 DEBG up_ds_listen was notified
ok
Sep 22 23:21:37.848 DEBG up_ds_listen process 1000
Sep 22 23:21:37.848 DEBG [2] Read already AckReady 1005, : downstairs
Sep 22 23:21:37.848 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.848 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51964, task: repair
Sep 22 23:21:37.848 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.848 DEBG up_ds_listen was notified
Sep 22 23:21:37.848 INFO Using repair address: 127.0.0.1:51964, task: main
Sep 22 23:21:37.848 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.848 DEBG up_ds_listen process 1005
Sep 22 23:21:37.848 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:21:37.848 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.848 INFO Created new region file "/tmp/downstairs-6NhFd3Sv/region.json"
Sep 22 23:21:37.849 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.849 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.849 INFO Opened existing region file "/tmp/downstairs-w5PkmiSc/region.json"
Sep 22 23:21:37.849 INFO Database read version 1
Sep 22 23:21:37.849 INFO Database write version 1
Sep 22 23:21:37.850 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.850 INFO Upstairs starts
Sep 22 23:21:37.850 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.850 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.850 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.850 INFO Crucible stats registered with UUID: ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5
Sep 22 23:21:37.850 INFO Crucible ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 has session id: f0462090-a03a-49ad-a727-20f1717992bc
Sep 22 23:21:37.850 DEBG Flush     :1003 extent_limit None deps:[JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.850 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.850 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.850 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.850 INFO [0] connecting to 127.0.0.1:46928, looper: 0
Sep 22 23:21:37.850 DEBG up_ds_listen was notified
Sep 22 23:21:37.850 INFO [1] connecting to 127.0.0.1:43209, looper: 1
Sep 22 23:21:37.850 DEBG up_ds_listen process 1003
Sep 22 23:21:37.850 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:37.850 INFO UUID: 32384725-b4ea-4f5d-a53b-d921af2e648c
Sep 22 23:21:37.850 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.850 INFO [2] connecting to 127.0.0.1:44686, looper: 2
Sep 22 23:21:37.850 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.850 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.850 INFO Request to deactivate this guest
Sep 22 23:21:37.850 INFO Using address: 127.0.0.1:45589, task: main
Sep 22 23:21:37.850 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.850 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 set deactivating.
Sep 22 23:21:37.850 DEBG [rc] retire 1003 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003)], : downstairs
Sep 22 23:21:37.850 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:21:37.850 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.850 INFO Flush timeout: 0.5
Sep 22 23:21:37.850 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.851 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.851 INFO [1] ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 looper connected, looper: 1
Sep 22 23:21:37.851 INFO [1] Proc runs for 127.0.0.1:43209 in state New
Sep 22 23:21:37.851 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33220, task: repair
Sep 22 23:21:37.851 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33220, task: repair
Sep 22 23:21:37.851 DEBG IO Read  1004 has deps []
Sep 22 23:21:37.851 INFO listening, local_addr: 127.0.0.1:33220, task: repair
Sep 22 23:21:37.851 INFO accepted connection from 127.0.0.1:42260, task: main
Sep 22 23:21:37.851 INFO [2] ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 looper connected, looper: 2
Sep 22 23:21:37.851 INFO [2] Proc runs for 127.0.0.1:44686 in state New
Sep 22 23:21:37.851 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33220, task: repair
Sep 22 23:21:37.851 INFO Using repair address: 127.0.0.1:33220, task: main
Sep 22 23:21:37.851 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.851 INFO accepted connection from 127.0.0.1:37386, task: main
Sep 22 23:21:37.851 INFO accepted connection from 127.0.0.1:33056, task: main
Sep 22 23:21:37.851 INFO [0] ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 looper connected, looper: 0
Sep 22 23:21:37.851 INFO [0] Proc runs for 127.0.0.1:46928 in state New
Sep 22 23:21:37.852 DEBG Read      :1004 deps:[] res:true
Sep 22 23:21:37.852 INFO Connection request from ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 with version 4, task: proc
Sep 22 23:21:37.852 INFO upstairs UpstairsConnection { upstairs_id: ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5, session_id: 9d285066-2c55-4273-bd21-6cddb01ea677, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.852 INFO Connection request from ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 with version 4, task: proc
Sep 22 23:21:37.852 INFO upstairs UpstairsConnection { upstairs_id: ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5, session_id: 9d285066-2c55-4273-bd21-6cddb01ea677, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.852 INFO Connection request from ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 with version 4, task: proc
Sep 22 23:21:37.852 INFO upstairs UpstairsConnection { upstairs_id: ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5, session_id: 9d285066-2c55-4273-bd21-6cddb01ea677, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.852 INFO [1] ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 (9d285066-2c55-4273-bd21-6cddb01ea677) New New New ds_transition to WaitActive
Sep 22 23:21:37.852 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.852 DEBG Read      :1004 deps:[] res:true
Sep 22 23:21:37.853 INFO [2] ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 (9d285066-2c55-4273-bd21-6cddb01ea677) New WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.853 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.853 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:37.853 INFO [0] ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 (9d285066-2c55-4273-bd21-6cddb01ea677) New WaitActive WaitActive ds_transition to WaitActive
Sep 22 23:21:37.853 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.853 INFO Upstairs starts
The guest has requested activation
Sep 22 23:21:37.853 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.853 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.853 INFO ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 active request set
Sep 22 23:21:37.853 INFO Crucible stats registered with UUID: 91b6bfa9-6f83-4cc0-a735-385db0454268
Sep 22 23:21:37.853 DEBG Read      :1004 deps:[] res:true
Sep 22 23:21:37.853 INFO Crucible 91b6bfa9-6f83-4cc0-a735-385db0454268 has session id: 240e8c63-755c-4bf6-a701-d7f79a8c0d23
Sep 22 23:21:37.853 INFO [0] received activate with gen 1
Sep 22 23:21:37.853 INFO [0] client got ds_active_rx, promote! session 9d285066-2c55-4273-bd21-6cddb01ea677
Sep 22 23:21:37.853 INFO [1] received activate with gen 1
Sep 22 23:21:37.853 INFO [1] client got ds_active_rx, promote! session 9d285066-2c55-4273-bd21-6cddb01ea677
Sep 22 23:21:37.853 INFO [2] received activate with gen 1
Sep 22 23:21:37.853 INFO [2] client got ds_active_rx, promote! session 9d285066-2c55-4273-bd21-6cddb01ea677
Sep 22 23:21:37.853 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.853 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.853 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.853 INFO UpstairsConnection { upstairs_id: ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5, session_id: 9d285066-2c55-4273-bd21-6cddb01ea677, gen: 1 } is now active (read-write)
Sep 22 23:21:37.853 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.853 INFO Opened existing region file "/tmp/downstairs-6NhFd3Sv/region.json"
Sep 22 23:21:37.853 INFO Database read version 1
Sep 22 23:21:37.853 INFO [0] connecting to 127.0.0.1:34995, looper: 0
Sep 22 23:21:37.853 INFO Database write version 1
Sep 22 23:21:37.853 INFO UpstairsConnection { upstairs_id: ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5, session_id: 9d285066-2c55-4273-bd21-6cddb01ea677, gen: 1 } is now active (read-write)
Sep 22 23:21:37.854 INFO [1] connecting to 127.0.0.1:45660, looper: 1
Sep 22 23:21:37.854 INFO UpstairsConnection { upstairs_id: ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5, session_id: 9d285066-2c55-4273-bd21-6cddb01ea677, gen: 1 } is now active (read-write)
Sep 22 23:21:37.854 INFO [2] connecting to 127.0.0.1:45589, looper: 2
Sep 22 23:21:37.854 DEBG Write     :1000 deps:[] res:true
test test::integration_test_scrub_short ... ok
Sep 22 23:21:37.854 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.854 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.854 INFO Flush timeout: 0.5
Sep 22 23:21:37.854 INFO [1] downstairs client at 127.0.0.1:43209 has UUID eb9758aa-5126-4e8c-b409-22a2bcacbe1b
Sep 22 23:21:37.854 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: eb9758aa-5126-4e8c-b409-22a2bcacbe1b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.854 INFO ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.854 INFO [2] downstairs client at 127.0.0.1:44686 has UUID 8472a597-92db-4cea-a2e5-658ef1a3d615
Sep 22 23:21:37.854 INFO [2] 91b6bfa9-6f83-4cc0-a735-385db0454268 looper connected, looper: 2
Sep 22 23:21:37.854 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 8472a597-92db-4cea-a2e5-658ef1a3d615, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.854 INFO [2] Proc runs for 127.0.0.1:45589 in state New
Sep 22 23:21:37.854 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.854 INFO ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.854 INFO [0] 91b6bfa9-6f83-4cc0-a735-385db0454268 looper connected, looper: 0
Sep 22 23:21:37.854 INFO [0] downstairs client at 127.0.0.1:46928 has UUID dc8a7c43-ebc6-4a48-b659-c51804dc92bb
Sep 22 23:21:37.854 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: dc8a7c43-ebc6-4a48-b659-c51804dc92bb, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.854 INFO Created new region file "/tmp/downstairs-eu4aQhL5/region.json"
Sep 22 23:21:37.854 INFO [0] Proc runs for 127.0.0.1:34995 in state New
Sep 22 23:21:37.854 INFO UUID: 70aec45a-414c-484a-9aa8-e75f71202205
Sep 22 23:21:37.854 INFO ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.854 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.854 INFO [1] 91b6bfa9-6f83-4cc0-a735-385db0454268 looper connected, looper: 1
Sep 22 23:21:37.855 INFO [1] Proc runs for 127.0.0.1:45660 in state New
Sep 22 23:21:37.855 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.855 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.855 INFO Using address: 127.0.0.1:65399, task: main
Sep 22 23:21:37.855 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.855 INFO accepted connection from 127.0.0.1:51086, task: main
Sep 22 23:21:37.855 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:37.855 INFO accepted connection from 127.0.0.1:33129, task: main
Sep 22 23:21:37.855 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.855 INFO accepted connection from 127.0.0.1:34652, task: main
Sep 22 23:21:37.855 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.855 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.855 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.855 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:55978, task: repair
Sep 22 23:21:37.855 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:55978, task: repair
Sep 22 23:21:37.855 INFO Connection request from 91b6bfa9-6f83-4cc0-a735-385db0454268 with version 4, task: proc
Sep 22 23:21:37.855 INFO upstairs UpstairsConnection { upstairs_id: 91b6bfa9-6f83-4cc0-a735-385db0454268, session_id: b6befe75-491e-4821-85c4-60905223c3c2, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.855 INFO listening, local_addr: 127.0.0.1:55978, task: repair
Sep 22 23:21:37.855 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.855 INFO [1] deactivate job 1001 not InProgress flush, NO
Sep 22 23:21:37.855 INFO Connection request from 91b6bfa9-6f83-4cc0-a735-385db0454268 with version 4, task: proc
Sep 22 23:21:37.855 INFO [2] deactivate job 1001 not InProgress flush, NO
Sep 22 23:21:37.855 INFO upstairs UpstairsConnection { upstairs_id: 91b6bfa9-6f83-4cc0-a735-385db0454268, session_id: b6befe75-491e-4821-85c4-60905223c3c2, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.855 INFO [0] deactivate job 1001 not InProgress flush, NO
Sep 22 23:21:37.855 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:55978, task: repair
Sep 22 23:21:37.855 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.855 INFO Using repair address: 127.0.0.1:55978, task: main
Sep 22 23:21:37.855 INFO Connection request from 91b6bfa9-6f83-4cc0-a735-385db0454268 with version 4, task: proc
Sep 22 23:21:37.855 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.855 INFO upstairs UpstairsConnection { upstairs_id: 91b6bfa9-6f83-4cc0-a735-385db0454268, session_id: b6befe75-491e-4821-85c4-60905223c3c2, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.855 INFO [1] ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 (9d285066-2c55-4273-bd21-6cddb01ea677) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.856 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.856 WARN [1] new RM replaced this: None
Sep 22 23:21:37.856 INFO [1] Starts reconcile loop
The guest has requested activation
Sep 22 23:21:37.856 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.856 INFO [2] ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 (9d285066-2c55-4273-bd21-6cddb01ea677) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.856 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.856 WARN [2] new RM replaced this: None
Sep 22 23:21:37.856 INFO [2] Starts reconcile loop
Sep 22 23:21:37.856 INFO [2] 91b6bfa9-6f83-4cc0-a735-385db0454268 (b6befe75-491e-4821-85c4-60905223c3c2) New New New ds_transition to WaitActive
Sep 22 23:21:37.856 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.856 INFO Created new region file "/tmp/downstairs-CuiYNtKt/region.json"
Sep 22 23:21:37.856 INFO [0] ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 (9d285066-2c55-4273-bd21-6cddb01ea677) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.856 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.856 WARN [0] new RM replaced this: None
Sep 22 23:21:37.856 INFO [0] Starts reconcile loop
Sep 22 23:21:37.856 INFO [0] 91b6bfa9-6f83-4cc0-a735-385db0454268 (b6befe75-491e-4821-85c4-60905223c3c2) New New WaitActive ds_transition to WaitActive
Sep 22 23:21:37.856 INFO [1] 127.0.0.1:43209 task reports connection:true
Sep 22 23:21:37.856 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.856 INFO ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.856 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.856 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.856 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.856 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.856 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.856 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.856 INFO [1] 91b6bfa9-6f83-4cc0-a735-385db0454268 (b6befe75-491e-4821-85c4-60905223c3c2) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:21:37.856 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.856 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.856 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.856 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.856 INFO Max found gen is 1
Sep 22 23:21:37.856 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.856 INFO Next flush: 1
Sep 22 23:21:37.856 INFO All extents match
Sep 22 23:21:37.856 INFO No downstairs repair required
Sep 22 23:21:37.856 INFO No initial repair work was required
Sep 22 23:21:37.856 INFO 91b6bfa9-6f83-4cc0-a735-385db0454268 active request set
Sep 22 23:21:37.856 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.856 INFO ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 is now active with session: 9d285066-2c55-4273-bd21-6cddb01ea677
Sep 22 23:21:37.856 INFO ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 Set Active after no repair
Sep 22 23:21:37.856 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.856 INFO [0] received activate with gen 1
Sep 22 23:21:37.856 INFO Set check for repair
Sep 22 23:21:37.856 INFO [0] client got ds_active_rx, promote! session b6befe75-491e-4821-85c4-60905223c3c2
Sep 22 23:21:37.856 INFO [2] 127.0.0.1:44686 task reports connection:true
Sep 22 23:21:37.856 INFO ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 Active Active Active
Sep 22 23:21:37.856 INFO Set check for repair
Sep 22 23:21:37.856 INFO [0] 127.0.0.1:46928 task reports connection:true
Sep 22 23:21:37.856 INFO ea4527bd-0243-45b8-8cd6-7dddcfc8c8f5 Active Active Active
Sep 22 23:21:37.856 INFO [1] received activate with gen 1
Sep 22 23:21:37.856 INFO Set check for repair
Sep 22 23:21:37.856 INFO [1] client got ds_active_rx, promote! session b6befe75-491e-4821-85c4-60905223c3c2
Sep 22 23:21:37.856 INFO [0] received reconcile message
Sep 22 23:21:37.856 INFO [0] All repairs completed, exit
Sep 22 23:21:37.856 INFO [0] Starts cmd_loop
Sep 22 23:21:37.856 INFO [2] received activate with gen 1
Sep 22 23:21:37.856 INFO [2] client got ds_active_rx, promote! session b6befe75-491e-4821-85c4-60905223c3c2
Sep 22 23:21:37.856 INFO [1] received reconcile message
Sep 22 23:21:37.856 INFO [1] All repairs completed, exit
Sep 22 23:21:37.856 INFO [1] Starts cmd_loop
Sep 22 23:21:37.856 INFO [2] received reconcile message
Sep 22 23:21:37.856 INFO UpstairsConnection { upstairs_id: 91b6bfa9-6f83-4cc0-a735-385db0454268, session_id: b6befe75-491e-4821-85c4-60905223c3c2, gen: 1 } is now active (read-write)
Sep 22 23:21:37.856 INFO [2] All repairs completed, exit
Sep 22 23:21:37.857 INFO [2] Starts cmd_loop
Sep 22 23:21:37.857 INFO UpstairsConnection { upstairs_id: 91b6bfa9-6f83-4cc0-a735-385db0454268, session_id: b6befe75-491e-4821-85c4-60905223c3c2, gen: 1 } is now active (read-write)
The guest has finished waiting for activation
Sep 22 23:21:37.857 DEBG [0] Read AckReady 1004, : downstairs
Sep 22 23:21:37.857 INFO UpstairsConnection { upstairs_id: 91b6bfa9-6f83-4cc0-a735-385db0454268, session_id: b6befe75-491e-4821-85c4-60905223c3c2, gen: 1 } is now active (read-write)
Sep 22 23:21:37.857 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.857 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.857 INFO [2] downstairs client at 127.0.0.1:45589 has UUID 32384725-b4ea-4f5d-a53b-d921af2e648c
Sep 22 23:21:37.857 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.857 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 32384725-b4ea-4f5d-a53b-d921af2e648c, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.857 INFO 91b6bfa9-6f83-4cc0-a735-385db0454268 WaitActive WaitActive WaitActive
Sep 22 23:21:37.857 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:37.857 INFO [0] downstairs client at 127.0.0.1:34995 has UUID dc9055ff-ffac-4e79-bd57-721a805ccf83
Sep 22 23:21:37.857 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: dc9055ff-ffac-4e79-bd57-721a805ccf83, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.857 INFO 91b6bfa9-6f83-4cc0-a735-385db0454268 WaitActive WaitActive WaitActive
Sep 22 23:21:37.858 INFO [1] downstairs client at 127.0.0.1:45660 has UUID 2e1bc31f-575d-410d-bb06-222733a39c1e
Sep 22 23:21:37.858 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2e1bc31f-575d-410d-bb06-222733a39c1e, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.858 INFO [1] check deactivate YES
Sep 22 23:21:37.858 INFO 91b6bfa9-6f83-4cc0-a735-385db0454268 WaitActive WaitActive WaitActive
Sep 22 23:21:37.858 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (c9717b37-7d6a-40f7-9030-e7e57914f334) Active Active Active ds_transition to Deactivated
Sep 22 23:21:37.858 INFO [1] Transition from Active to Deactivated
Sep 22 23:21:37.858 INFO [2] check deactivate YES
Sep 22 23:21:37.858 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (c9717b37-7d6a-40f7-9030-e7e57914f334) Active Deactivated Active ds_transition to Deactivated
Sep 22 23:21:37.858 INFO [2] Transition from Active to Deactivated
Sep 22 23:21:37.858 DEBG [0] deactivate flush 1001 done, : downstairs
Sep 22 23:21:37.858 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.858 INFO [0] check deactivate YES
Sep 22 23:21:37.858 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.858 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (c9717b37-7d6a-40f7-9030-e7e57914f334) Active Deactivated Deactivated ds_transition to Deactivated
Sep 22 23:21:37.858 INFO [0] Transition from Active to Deactivated
Sep 22 23:21:37.858 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.858 INFO Opened existing region file "/tmp/downstairs-eu4aQhL5/region.json"
Sep 22 23:21:37.858 INFO Database read version 1
Sep 22 23:21:37.858 ERRO 127.0.0.1:44628: proc: [1] client work task ended, Ok(Err([1] exits after deactivation)), so we end too, looper: 1
Sep 22 23:21:37.858 INFO Database write version 1
Sep 22 23:21:37.858 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.858 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Gone missing, transition from Deactivated to New
Sep 22 23:21:37.858 INFO deactivate transition checking...
Sep 22 23:21:37.858 INFO deactivate_transition Deactivated NO
Sep 22 23:21:37.858 INFO deactivate_transition New Maybe 
Sep 22 23:21:37.858 INFO deactivate_transition Deactivated NO
Sep 22 23:21:37.858 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 connection to 127.0.0.1:44628 closed, looper: 1
Sep 22 23:21:37.858 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.858 ERRO 127.0.0.1:38317: proc: [2] client work task ended, Ok(Err([2] exits after deactivation)), so we end too, looper: 2
Sep 22 23:21:37.858 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.858 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Gone missing, transition from Deactivated to New
Sep 22 23:21:37.858 INFO deactivate transition checking...
Sep 22 23:21:37.858 INFO deactivate_transition Deactivated NO
Sep 22 23:21:37.858 INFO deactivate_transition New Maybe 
Sep 22 23:21:37.858 INFO deactivate_transition New Maybe 
Sep 22 23:21:37.858 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 connection to 127.0.0.1:38317 closed, looper: 2
Sep 22 23:21:37.858 DEBG up_ds_listen was notified
Sep 22 23:21:37.858 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.858 DEBG up_ds_listen process 1001
Sep 22 23:21:37.858 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.859 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.859 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:21:37.859 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.859 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.859 ERRO 127.0.0.1:65137: proc: [0] client work task ended, Ok(Err([0] exits after deactivation)), so we end too, looper: 0
Sep 22 23:21:37.859 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Gone missing, transition from Deactivated to New
Sep 22 23:21:37.859 INFO deactivate transition checking...
Sep 22 23:21:37.859 INFO deactivate_transition New Maybe 
Sep 22 23:21:37.859 INFO deactivate_transition New Maybe 
Sep 22 23:21:37.859 INFO deactivate_transition New Maybe 
Sep 22 23:21:37.859 INFO All DS in the proper state! -> INIT
Sep 22 23:21:37.859 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 connection to 127.0.0.1:65137 closed, looper: 0
Sep 22 23:21:37.859 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.859 INFO [1] 127.0.0.1:44628 task reports connection:false
Sep 22 23:21:37.859 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 New New New
Sep 22 23:21:37.859 INFO [1] 127.0.0.1:44628 task reports offline
Sep 22 23:21:37.859 INFO [2] 127.0.0.1:38317 task reports connection:false
Sep 22 23:21:37.859 INFO [2] 91b6bfa9-6f83-4cc0-a735-385db0454268 (b6befe75-491e-4821-85c4-60905223c3c2) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.859 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 New New New
Sep 22 23:21:37.859 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.859 INFO [2] 127.0.0.1:38317 task reports offline
Sep 22 23:21:37.859 WARN [2] new RM replaced this: None
Sep 22 23:21:37.859 INFO [0] 127.0.0.1:65137 task reports connection:false
Sep 22 23:21:37.859 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 New New New
Sep 22 23:21:37.859 INFO [2] Starts reconcile loop
Sep 22 23:21:37.859 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.859 INFO [0] 127.0.0.1:65137 task reports offline
Sep 22 23:21:37.859 INFO Opened existing region file "/tmp/downstairs-CuiYNtKt/region.json"
Sep 22 23:21:37.859 INFO Database read version 1
Sep 22 23:21:37.859 INFO Database write version 1
Sep 22 23:21:37.859 INFO [0] 91b6bfa9-6f83-4cc0-a735-385db0454268 (b6befe75-491e-4821-85c4-60905223c3c2) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.859 DEBG [2] Read already AckReady 1004, : downstairs
Sep 22 23:21:37.859 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.859 WARN [0] new RM replaced this: None
Sep 22 23:21:37.859 INFO [0] Starts reconcile loop
Sep 22 23:21:37.859 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.859 INFO Opened existing region file "/tmp/downstairs-bxfe8IV2/region.json"
Sep 22 23:21:37.859 DEBG [1] Read AckReady 1000, : downstairs
Sep 22 23:21:37.859 INFO Database read version 1
Sep 22 23:21:37.859 INFO [1] 91b6bfa9-6f83-4cc0-a735-385db0454268 (b6befe75-491e-4821-85c4-60905223c3c2) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.859 INFO Database write version 1
Sep 22 23:21:37.859 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.859 WARN [1] new RM replaced this: None
Sep 22 23:21:37.859 INFO [1] Starts reconcile loop
Sep 22 23:21:37.859 INFO [2] 127.0.0.1:45589 task reports connection:true
Sep 22 23:21:37.859 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.859 INFO 91b6bfa9-6f83-4cc0-a735-385db0454268 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.859 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.860 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.860 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.860 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.860 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.860 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.860 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.860 DEBG [0] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.860 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.860 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.860 INFO Max found gen is 1
Sep 22 23:21:37.860 DEBG up_ds_listen was notified
Sep 22 23:21:37.860 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.860 DEBG up_ds_listen process 1000
Sep 22 23:21:37.860 INFO Next flush: 1
Sep 22 23:21:37.860 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.860 INFO All extents match
Sep 22 23:21:37.860 INFO No downstairs repair required
Sep 22 23:21:37.860 INFO No initial repair work was required
Sep 22 23:21:37.860 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.860 INFO 91b6bfa9-6f83-4cc0-a735-385db0454268 is now active with session: b6befe75-491e-4821-85c4-60905223c3c2
Sep 22 23:21:37.860 INFO 91b6bfa9-6f83-4cc0-a735-385db0454268 Set Active after no repair
Sep 22 23:21:37.860 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.860 INFO Set check for repair
Sep 22 23:21:37.860 INFO [0] 127.0.0.1:34995 task reports connection:true
Sep 22 23:21:37.860 INFO 91b6bfa9-6f83-4cc0-a735-385db0454268 Active Active Active
Sep 22 23:21:37.860 INFO Set check for repair
Sep 22 23:21:37.860 INFO [1] 127.0.0.1:45660 task reports connection:true
Sep 22 23:21:37.860 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.860 INFO UUID: c731e991-4a16-42af-828a-390914aa5a24
Sep 22 23:21:37.860 INFO 91b6bfa9-6f83-4cc0-a735-385db0454268 Active Active Active
Sep 22 23:21:37.860 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.860 INFO Set check for repair
Sep 22 23:21:37.860 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.860 INFO [0] received reconcile message
Sep 22 23:21:37.860 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.860 INFO [0] All repairs completed, exit
Sep 22 23:21:37.860 INFO Using address: 127.0.0.1:64599, task: main
Sep 22 23:21:37.860 INFO [0] Starts cmd_loop
Sep 22 23:21:37.860 INFO [1] received reconcile message
Sep 22 23:21:37.860 INFO [1] All repairs completed, exit
Sep 22 23:21:37.860 INFO [1] Starts cmd_loop
Sep 22 23:21:37.860 INFO [2] received reconcile message
Sep 22 23:21:37.860 INFO [2] All repairs completed, exit
Sep 22 23:21:37.860 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:37.861 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.861 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52902, task: repair
Sep 22 23:21:37.861 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52902, task: repair
Sep 22 23:21:37.861 INFO listening, local_addr: 127.0.0.1:52902, task: repair
Sep 22 23:21:37.861 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.861 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52902, task: repair
Sep 22 23:21:37.861 INFO Using repair address: 127.0.0.1:52902, task: main
Sep 22 23:21:37.861 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.861 INFO UUID: 136e8694-ad8f-4812-bbb6-93a5da7b3a3e
Sep 22 23:21:37.861 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.861 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.861 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.861 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.861 INFO Using address: 127.0.0.1:54774, task: main
Sep 22 23:21:37.861 INFO Created new region file "/tmp/downstairs-UPdbxzP8/region.json"
Sep 22 23:21:37.862 INFO UUID: f5ea863d-974b-4712-8f07-7666c185979b
Sep 22 23:21:37.862 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.862 DEBG [1] Read already AckReady 1004, : downstairs
Sep 22 23:21:37.862 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.862 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.862 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.862 DEBG up_ds_listen was notified
Sep 22 23:21:37.862 INFO Using address: 127.0.0.1:48404, task: main
Sep 22 23:21:37.862 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.862 DEBG up_ds_listen process 1004
Sep 22 23:21:37.862 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:21:37.862 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51281, task: repair
Sep 22 23:21:37.862 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51281, task: repair
Sep 22 23:21:37.862 INFO listening, local_addr: 127.0.0.1:51281, task: repair
Sep 22 23:21:37.862 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51281, task: repair
Sep 22 23:21:37.862 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.862 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.862 INFO Using repair address: 127.0.0.1:51281, task: main
Sep 22 23:21:37.862 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.862 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33274, task: repair
Sep 22 23:21:37.862 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33274, task: repair
Sep 22 23:21:37.862 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.862 INFO listening, local_addr: 127.0.0.1:33274, task: repair
Sep 22 23:21:37.862 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.863 INFO Created new region file "/tmp/downstairs-HaYagj8S/region.json"
Sep 22 23:21:37.863 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33274, task: repair
Sep 22 23:21:37.863 INFO Using repair address: 127.0.0.1:33274, task: main
Sep 22 23:21:37.863 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.863 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.863 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.863 INFO Opened existing region file "/tmp/downstairs-RQXy3Y00/region.json"
Sep 22 23:21:37.863 INFO Database read version 1
Sep 22 23:21:37.863 INFO Database write version 1
Sep 22 23:21:37.863 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:37.863 DEBG up_ds_listen was notified
Sep 22 23:21:37.863 DEBG up_ds_listen process 1001
Sep 22 23:21:37.863 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.863 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.864 DEBG [2] Read AckReady 1000, : downstairs
Sep 22 23:21:37.864 DEBG [0] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.864 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.864 DEBG up_ds_listen was notified
Sep 22 23:21:37.864 DEBG up_ds_listen process 1000
Sep 22 23:21:37.864 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.865 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.866 INFO UUID: a1447c8e-cf80-4bea-b730-010907ef0c7f
Sep 22 23:21:37.866 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.866 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.866 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.866 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.866 INFO Using address: 127.0.0.1:54505, task: main
test test::integration_test_scrub_useless ... ok
Sep 22 23:21:37.866 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.866 INFO Opened existing region file "/tmp/downstairs-UPdbxzP8/region.json"
Sep 22 23:21:37.866 INFO Database read version 1
Sep 22 23:21:37.866 INFO Database write version 1
Sep 22 23:21:37.867 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.867 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.867 INFO Created new region file "/tmp/downstairs-gL8cXuVs/region.json"
Sep 22 23:21:37.867 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36554, task: repair
Sep 22 23:21:37.867 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36554, task: repair
Sep 22 23:21:37.867 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.867 INFO listening, local_addr: 127.0.0.1:36554, task: repair
Sep 22 23:21:37.867 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36554, task: repair
Sep 22 23:21:37.867 INFO Using repair address: 127.0.0.1:36554, task: main
Sep 22 23:21:37.867 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.867 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.867 INFO Opened existing region file "/tmp/downstairs-agB7RjDz/region.json"
Sep 22 23:21:37.867 INFO Database read version 1
Sep 22 23:21:37.867 INFO Database write version 1
Sep 22 23:21:37.868 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.868 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.868 INFO Opened existing region file "/tmp/downstairs-HaYagj8S/region.json"
Sep 22 23:21:37.868 INFO Database read version 1
Sep 22 23:21:37.868 INFO Database write version 1
Sep 22 23:21:37.868 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:21:37.868 INFO UUID: e615c980-e0be-48be-967d-b06ac35abc56
Sep 22 23:21:37.868 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.868 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.868 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.868 INFO Using address: 127.0.0.1:62465, task: main
Sep 22 23:21:37.868 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.869 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49827, task: repair
Sep 22 23:21:37.869 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49827, task: repair
Sep 22 23:21:37.869 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:37.869 INFO listening, local_addr: 127.0.0.1:49827, task: repair
Sep 22 23:21:37.869 DEBG up_ds_listen was notified
Sep 22 23:21:37.869 DEBG up_ds_listen process 1001
Sep 22 23:21:37.869 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.869 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.869 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49827, task: repair
Sep 22 23:21:37.869 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.869 INFO Using repair address: 127.0.0.1:49827, task: main
Sep 22 23:21:37.869 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.869 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.870 INFO UUID: 1b0c673f-012e-4936-8109-087022151153
Sep 22 23:21:37.870 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.870 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.870 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.870 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.870 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.870 INFO Created new region file "/tmp/downstairs-AeRxtKo6/region.json"
Sep 22 23:21:37.870 INFO Using address: 127.0.0.1:38410, task: main
Sep 22 23:21:37.870 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.870 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.870 INFO Opened existing region file "/tmp/downstairs-gL8cXuVs/region.json"
Sep 22 23:21:37.870 INFO Database read version 1
Sep 22 23:21:37.870 INFO Database write version 1
Sep 22 23:21:37.870 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36658, task: repair
Sep 22 23:21:37.870 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36658, task: repair
Sep 22 23:21:37.870 INFO UUID: fee4220d-c596-4fac-bca8-6109f63683be
Sep 22 23:21:37.870 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.870 INFO listening, local_addr: 127.0.0.1:36658, task: repair
Sep 22 23:21:37.870 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.870 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.871 INFO Using address: 127.0.0.1:34215, task: main
Sep 22 23:21:37.871 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36658, task: repair
Sep 22 23:21:37.871 INFO Using repair address: 127.0.0.1:36658, task: main
Sep 22 23:21:37.871 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.871 INFO Upstairs starts
Sep 22 23:21:37.871 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.871 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.871 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.871 INFO Crucible stats registered with UUID: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5
Sep 22 23:21:37.871 INFO Crucible 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 has session id: 71cfa847-8b3e-4c1a-ab28-9b5cea9dd9a5
Sep 22 23:21:37.871 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49207, task: repair
Sep 22 23:21:37.871 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49207, task: repair
Sep 22 23:21:37.871 INFO listening, local_addr: 127.0.0.1:49207, task: repair
Sep 22 23:21:37.871 WARN upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:21:37.871 WARN upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } was previously active, clearing, task: main
Sep 22 23:21:37.871 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49207, task: repair
Sep 22 23:21:37.871 INFO connection (127.0.0.1:43896): all done
Sep 22 23:21:37.871 INFO Using repair address: 127.0.0.1:49207, task: main
Sep 22 23:21:37.871 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.871 WARN upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:21:37.871 WARN upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } was previously active, clearing, task: main
Sep 22 23:21:37.871 INFO connection (127.0.0.1:47893): all done
Sep 22 23:21:37.871 WARN upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:21:37.872 WARN upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: c9717b37-7d6a-40f7-9030-e7e57914f334, gen: 1 } was previously active, clearing, task: main
Sep 22 23:21:37.872 INFO connection (127.0.0.1:48011): all done
Sep 22 23:21:37.872 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.872 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.872 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.872 INFO [0] connecting to 127.0.0.1:48404, looper: 0
Sep 22 23:21:37.872 INFO [1] connecting to 127.0.0.1:54505, looper: 1
Sep 22 23:21:37.872 INFO [2] connecting to 127.0.0.1:38410, looper: 2
Sep 22 23:21:37.872 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.872 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.872 INFO Flush timeout: 0.5
Sep 22 23:21:37.872 INFO UUID: f146ddc8-437e-48f6-aa56-398321fa6a78
Sep 22 23:21:37.872 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.872 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.872 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.872 INFO Using address: 127.0.0.1:65200, task: main
Sep 22 23:21:37.873 INFO Upstairs starts
Sep 22 23:21:37.873 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.873 INFO accepted connection from 127.0.0.1:50540, task: main
Sep 22 23:21:37.873 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.873 INFO Crucible stats registered with UUID: 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d
Sep 22 23:21:37.873 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 looper connected, looper: 1
Sep 22 23:21:37.873 INFO Crucible 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d has session id: 9b529882-d0fe-4613-b03a-e8885917f615
Sep 22 23:21:37.873 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.873 INFO [1] Proc runs for 127.0.0.1:54505 in state New
Sep 22 23:21:37.873 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 looper connected, looper: 2
Sep 22 23:21:37.873 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.873 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36525, task: repair
Sep 22 23:21:37.873 INFO [2] Proc runs for 127.0.0.1:38410 in state New
Sep 22 23:21:37.873 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.873 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36525, task: repair
Sep 22 23:21:37.873 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.873 INFO listening, local_addr: 127.0.0.1:36525, task: repair
Sep 22 23:21:37.873 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.873 INFO accepted connection from 127.0.0.1:51558, task: main
Sep 22 23:21:37.873 INFO [0] connecting to 127.0.0.1:65399, looper: 0
Sep 22 23:21:37.873 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 looper connected, looper: 0
Sep 22 23:21:37.873 INFO [0] Proc runs for 127.0.0.1:48404 in state New
Sep 22 23:21:37.873 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36525, task: repair
Sep 22 23:21:37.873 INFO Using repair address: 127.0.0.1:36525, task: main
Sep 22 23:21:37.873 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.873 INFO accepted connection from 127.0.0.1:34528, task: main
Sep 22 23:21:37.873 INFO [1] connecting to 127.0.0.1:54774, looper: 1
Sep 22 23:21:37.873 INFO [2] connecting to 127.0.0.1:34215, looper: 2
Sep 22 23:21:37.873 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.874 INFO Connection request from 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 with version 4, task: proc
Sep 22 23:21:37.874 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.874 INFO upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 8442ae1d-103c-45db-9a72-ab1fbfb096dd, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:37.874 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.874 INFO Opened existing region file "/tmp/downstairs-AeRxtKo6/region.json"
Sep 22 23:21:37.874 INFO Flush timeout: 0.5
Sep 22 23:21:37.874 INFO Database read version 1
Sep 22 23:21:37.874 INFO Database write version 1
Sep 22 23:21:37.873 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.874 INFO Connection request from 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 with version 4, task: proc
Sep 22 23:21:37.874 INFO upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 8442ae1d-103c-45db-9a72-ab1fbfb096dd, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:37.874 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.874 INFO Created new region file "/tmp/downstairs-NWKRhaFi/region.json"
Sep 22 23:21:37.874 INFO Connection request from 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 with version 4, task: proc
Sep 22 23:21:37.874 INFO upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 8442ae1d-103c-45db-9a72-ab1fbfb096dd, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:37.874 INFO [2] 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d looper connected, looper: 2
Sep 22 23:21:37.874 INFO [2] Proc runs for 127.0.0.1:34215 in state New
Sep 22 23:21:37.874 INFO accepted connection from 127.0.0.1:56564, task: main
The guest has requested activation
Sep 22 23:21:37.874 DEBG [1] Read AckReady 1002, : downstairs
Sep 22 23:21:37.874 INFO accepted connection from 127.0.0.1:61438, task: main
Sep 22 23:21:37.874 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) New New New ds_transition to WaitActive
Sep 22 23:21:37.874 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.874 INFO accepted connection from 127.0.0.1:62233, task: main
Sep 22 23:21:37.874 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) New WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.874 INFO [0] 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d looper connected, looper: 0
Sep 22 23:21:37.874 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.874 INFO [0] Proc runs for 127.0.0.1:65399 in state New
Sep 22 23:21:37.874 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) New WaitActive WaitActive ds_transition to WaitActive
Sep 22 23:21:37.874 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.874 INFO [1] 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d looper connected, looper: 1
Sep 22 23:21:37.874 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.874 INFO [1] Proc runs for 127.0.0.1:54774 in state New
Sep 22 23:21:37.874 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 active request set
Sep 22 23:21:37.875 INFO [0] received activate with gen 2
Sep 22 23:21:37.875 INFO [0] client got ds_active_rx, promote! session 8442ae1d-103c-45db-9a72-ab1fbfb096dd
Sep 22 23:21:37.875 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:21:37.875 INFO [1] received activate with gen 2
Sep 22 23:21:37.875 INFO [1] client got ds_active_rx, promote! session 8442ae1d-103c-45db-9a72-ab1fbfb096dd
Sep 22 23:21:37.875 INFO Connection request from 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d with version 4, task: proc
Sep 22 23:21:37.875 INFO upstairs UpstairsConnection { upstairs_id: 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d, session_id: 39026538-33df-47fa-a1cb-bac551559eaf, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.875 INFO [2] received activate with gen 2
Sep 22 23:21:37.875 INFO [2] client got ds_active_rx, promote! session 8442ae1d-103c-45db-9a72-ab1fbfb096dd
Sep 22 23:21:37.875 INFO Connection request from 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d with version 4, task: proc
Sep 22 23:21:37.875 INFO upstairs UpstairsConnection { upstairs_id: 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d, session_id: 39026538-33df-47fa-a1cb-bac551559eaf, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.875 INFO Connection request from 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d with version 4, task: proc
Sep 22 23:21:37.875 INFO upstairs UpstairsConnection { upstairs_id: 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d, session_id: 39026538-33df-47fa-a1cb-bac551559eaf, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:21:37.875 INFO [2] 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d (39026538-33df-47fa-a1cb-bac551559eaf) New New New ds_transition to WaitActive
Sep 22 23:21:37.875 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.875 INFO [0] 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d (39026538-33df-47fa-a1cb-bac551559eaf) New New WaitActive ds_transition to WaitActive
Sep 22 23:21:37.875 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.875 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.875 INFO [1] 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d (39026538-33df-47fa-a1cb-bac551559eaf) WaitActive New WaitActive ds_transition to WaitActive
Sep 22 23:21:37.875 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.875 INFO 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d active request set
Sep 22 23:21:37.875 INFO [1] downstairs client at 127.0.0.1:54505 has UUID a1447c8e-cf80-4bea-b730-010907ef0c7f
Sep 22 23:21:37.875 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a1447c8e-cf80-4bea-b730-010907ef0c7f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.876 INFO [0] received activate with gen 1
Sep 22 23:21:37.876 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.876 INFO [0] client got ds_active_rx, promote! session 39026538-33df-47fa-a1cb-bac551559eaf
Sep 22 23:21:37.876 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.876 INFO UUID: 8e6d336f-2489-4c93-9267-791b6b43d94e
Sep 22 23:21:37.876 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.876 INFO [1] received activate with gen 1
Sep 22 23:21:37.876 INFO [2] downstairs client at 127.0.0.1:38410 has UUID 1b0c673f-012e-4936-8109-087022151153
Sep 22 23:21:37.876 INFO [1] client got ds_active_rx, promote! session 39026538-33df-47fa-a1cb-bac551559eaf
Sep 22 23:21:37.876 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1b0c673f-012e-4936-8109-087022151153, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.876 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.876 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.876 INFO Using address: 127.0.0.1:56505, task: main
Sep 22 23:21:37.876 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.876 INFO [2] received activate with gen 1
Sep 22 23:21:37.876 INFO [2] client got ds_active_rx, promote! session 39026538-33df-47fa-a1cb-bac551559eaf
Sep 22 23:21:37.876 INFO [0] downstairs client at 127.0.0.1:48404 has UUID f5ea863d-974b-4712-8f07-7666c185979b
Sep 22 23:21:37.876 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f5ea863d-974b-4712-8f07-7666c185979b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.876 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.876 INFO UpstairsConnection { upstairs_id: 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d, session_id: 39026538-33df-47fa-a1cb-bac551559eaf, gen: 1 } is now active (read-write)
Sep 22 23:21:37.876 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.876 INFO UpstairsConnection { upstairs_id: 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d, session_id: 39026538-33df-47fa-a1cb-bac551559eaf, gen: 1 } is now active (read-write)
Sep 22 23:21:37.876 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.876 INFO UpstairsConnection { upstairs_id: 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d, session_id: 39026538-33df-47fa-a1cb-bac551559eaf, gen: 1 } is now active (read-write)
Sep 22 23:21:37.876 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38700, task: repair
Sep 22 23:21:37.876 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38700, task: repair
Sep 22 23:21:37.876 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:37.876 INFO listening, local_addr: 127.0.0.1:38700, task: repair
Sep 22 23:21:37.877 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.877 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38700, task: repair
Sep 22 23:21:37.877 INFO Using repair address: 127.0.0.1:38700, task: main
Sep 22 23:21:37.877 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.877 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.877 INFO [2] downstairs client at 127.0.0.1:34215 has UUID fee4220d-c596-4fac-bca8-6109f63683be
Sep 22 23:21:37.877 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: fee4220d-c596-4fac-bca8-6109f63683be, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.877 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:37.877 INFO 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d WaitActive WaitActive WaitActive
Sep 22 23:21:37.877 INFO [0] downstairs client at 127.0.0.1:65399 has UUID 70aec45a-414c-484a-9aa8-e75f71202205
Sep 22 23:21:37.877 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 70aec45a-414c-484a-9aa8-e75f71202205, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.877 INFO 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d WaitActive WaitActive WaitActive
Sep 22 23:21:37.877 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.877 INFO [1] downstairs client at 127.0.0.1:54774 has UUID 136e8694-ad8f-4812-bbb6-93a5da7b3a3e
Sep 22 23:21:37.877 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 136e8694-ad8f-4812-bbb6-93a5da7b3a3e, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.877 INFO 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d WaitActive WaitActive WaitActive
Sep 22 23:21:37.877 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:37.878 INFO Upstairs starts
Sep 22 23:21:37.878 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.878 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.878 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.878 INFO Crucible stats registered with UUID: a334e2c5-02d5-47e7-a7d4-82e25a58ab9c
Sep 22 23:21:37.878 INFO Crucible a334e2c5-02d5-47e7-a7d4-82e25a58ab9c has session id: b24e6603-575e-47f3-85c3-9a1a62d154a4
Sep 22 23:21:37.878 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.878 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.878 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.878 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.878 INFO Opened existing region file "/tmp/downstairs-NWKRhaFi/region.json"
Sep 22 23:21:37.878 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.878 INFO Database read version 1
Sep 22 23:21:37.878 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.878 INFO Database write version 1
Sep 22 23:21:37.878 INFO [0] connecting to 127.0.0.1:64599, looper: 0
Sep 22 23:21:37.878 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.878 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.878 WARN [1] new RM replaced this: None
Sep 22 23:21:37.878 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.878 INFO [1] Starts reconcile loop
Sep 22 23:21:37.878 INFO [1] connecting to 127.0.0.1:62465, looper: 1
Sep 22 23:21:37.878 INFO [2] connecting to 127.0.0.1:56505, looper: 2
Sep 22 23:21:37.878 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.878 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.878 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.878 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.878 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.878 WARN [2] new RM replaced this: None
Sep 22 23:21:37.878 INFO Flush timeout: 0.5
Sep 22 23:21:37.878 INFO [2] Starts reconcile loop
Sep 22 23:21:37.878 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.878 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.878 WARN [0] new RM replaced this: None
Sep 22 23:21:37.878 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.878 INFO accepted connection from 127.0.0.1:44432, task: main
Sep 22 23:21:37.878 INFO [0] Starts reconcile loop
Sep 22 23:21:37.878 INFO [1] 127.0.0.1:54505 task reports connection:true
Sep 22 23:21:37.878 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.879 INFO [0]R  flush_numbers: [1, 1]
Sep 22 23:21:37.879 INFO accepted connection from 127.0.0.1:60279, task: main
Sep 22 23:21:37.879 INFO [0]R  generation: [1, 1]
Sep 22 23:21:37.879 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.879 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.879 INFO accepted connection from 127.0.0.1:52426, task: main
Sep 22 23:21:37.879 INFO [1]R  flush_numbers: [1, 1]
Sep 22 23:21:37.879 INFO [1]R  generation: [1, 1]
Sep 22 23:21:37.879 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.879 INFO [2]R  flush_numbers: [1, 1]
Sep 22 23:21:37.879 INFO [0] a334e2c5-02d5-47e7-a7d4-82e25a58ab9c looper connected, looper: 0
Sep 22 23:21:37.879 INFO [2]R  generation: [1, 1]
Sep 22 23:21:37.879 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.879 INFO [0] Proc runs for 127.0.0.1:64599 in state New
Sep 22 23:21:37.879 INFO Max found gen is 2
Sep 22 23:21:37.879 INFO Generation requested: 2 >= found:2
Sep 22 23:21:37.879 INFO Next flush: 2
Sep 22 23:21:37.879 INFO [1] a334e2c5-02d5-47e7-a7d4-82e25a58ab9c looper connected, looper: 1
Sep 22 23:21:37.879 INFO All extents match
Sep 22 23:21:37.879 INFO [2] 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d (39026538-33df-47fa-a1cb-bac551559eaf) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.879 INFO No downstairs repair required
Sep 22 23:21:37.879 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.879 INFO [1] Proc runs for 127.0.0.1:62465 in state New
Sep 22 23:21:37.879 INFO No initial repair work was required
Sep 22 23:21:37.879 WARN [2] new RM replaced this: None
Sep 22 23:21:37.879 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.879 INFO [2] a334e2c5-02d5-47e7-a7d4-82e25a58ab9c looper connected, looper: 2
Sep 22 23:21:37.879 INFO [2] Starts reconcile loop
Sep 22 23:21:37.879 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 is now active with session: 8442ae1d-103c-45db-9a72-ab1fbfb096dd
Sep 22 23:21:37.879 INFO [2] Proc runs for 127.0.0.1:56505 in state New
Sep 22 23:21:37.879 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Set Active after no repair
Sep 22 23:21:37.879 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.879 INFO Set check for repair
Sep 22 23:21:37.879 INFO [0] 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d (39026538-33df-47fa-a1cb-bac551559eaf) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.879 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.879 INFO [2] 127.0.0.1:38410 task reports connection:true
Sep 22 23:21:37.879 WARN [0] new RM replaced this: None
Sep 22 23:21:37.879 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Active Active Active
Sep 22 23:21:37.879 INFO [0] Starts reconcile loop
Sep 22 23:21:37.879 INFO Set check for repair
Sep 22 23:21:37.879 INFO [0] 127.0.0.1:48404 task reports connection:true
Sep 22 23:21:37.879 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Active Active Active
Sep 22 23:21:37.879 INFO [1] 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d (39026538-33df-47fa-a1cb-bac551559eaf) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum
Sep 22 23:21:37.879 INFO Set check for repair
Sep 22 23:21:37.879 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.879 INFO Connection request from a334e2c5-02d5-47e7-a7d4-82e25a58ab9c with version 4, task: proc
Sep 22 23:21:37.879 INFO upstairs UpstairsConnection { upstairs_id: a334e2c5-02d5-47e7-a7d4-82e25a58ab9c, session_id: 71f2b629-5d8b-450e-858e-3d0aaedbefc8, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.879 WARN [1] new RM replaced this: None
Sep 22 23:21:37.879 INFO [0] received reconcile message
Sep 22 23:21:37.879 INFO [1] Starts reconcile loop
Sep 22 23:21:37.879 INFO [0] All repairs completed, exit
Sep 22 23:21:37.879 INFO Connection request from a334e2c5-02d5-47e7-a7d4-82e25a58ab9c with version 4, task: proc
Sep 22 23:21:37.879 INFO upstairs UpstairsConnection { upstairs_id: a334e2c5-02d5-47e7-a7d4-82e25a58ab9c, session_id: 71f2b629-5d8b-450e-858e-3d0aaedbefc8, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.879 INFO [0] Starts cmd_loop
Sep 22 23:21:37.879 INFO [2] 127.0.0.1:34215 task reports connection:true
Sep 22 23:21:37.879 INFO 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.879 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.879 INFO Connection request from a334e2c5-02d5-47e7-a7d4-82e25a58ab9c with version 4, task: proc
Sep 22 23:21:37.879 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.879 INFO upstairs UpstairsConnection { upstairs_id: a334e2c5-02d5-47e7-a7d4-82e25a58ab9c, session_id: 71f2b629-5d8b-450e-858e-3d0aaedbefc8, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.879 INFO [1] received reconcile message
Sep 22 23:21:37.879 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.879 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.879 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.879 INFO [1] All repairs completed, exit
Sep 22 23:21:37.879 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.879 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.879 INFO [1] Starts cmd_loop
Sep 22 23:21:37.879 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.879 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.879 INFO Max found gen is 1
The guest has requested activation
Sep 22 23:21:37.879 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.879 INFO [2] received reconcile message
Sep 22 23:21:37.879 INFO Next flush: 1
Sep 22 23:21:37.879 INFO [2] All repairs completed, exit
Sep 22 23:21:37.879 INFO All extents match
Sep 22 23:21:37.879 INFO No downstairs repair required
Sep 22 23:21:37.879 INFO [0] a334e2c5-02d5-47e7-a7d4-82e25a58ab9c (71f2b629-5d8b-450e-858e-3d0aaedbefc8) New New New ds_transition to WaitActive
Sep 22 23:21:37.879 INFO [2] Starts cmd_loop
Sep 22 23:21:37.880 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.880 INFO No initial repair work was required
Sep 22 23:21:37.880 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.880 INFO 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d is now active with session: 39026538-33df-47fa-a1cb-bac551559eaf
Sep 22 23:21:37.880 INFO [1] a334e2c5-02d5-47e7-a7d4-82e25a58ab9c (71f2b629-5d8b-450e-858e-3d0aaedbefc8) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.880 INFO 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d Set Active after no repair
Sep 22 23:21:37.880 INFO [1] Transition from New to WaitActive
The guest has finished waiting for activation
Sep 22 23:21:37.880 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.880 INFO Set check for repair
Sep 22 23:21:37.880 INFO [2] a334e2c5-02d5-47e7-a7d4-82e25a58ab9c (71f2b629-5d8b-450e-858e-3d0aaedbefc8) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.880 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.880 INFO [0] 127.0.0.1:65399 task reports connection:true
Sep 22 23:21:37.880 INFO 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d Active Active Active
Sep 22 23:21:37.880 DEBG [0] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.880 INFO Set check for repair
Sep 22 23:21:37.880 INFO a334e2c5-02d5-47e7-a7d4-82e25a58ab9c active request set
Sep 22 23:21:37.880 INFO [1] 127.0.0.1:54774 task reports connection:true
Sep 22 23:21:37.880 INFO 1ec1b14d-7169-4b28-b3ed-c4eece97cf0d Active Active Active
Sep 22 23:21:37.880 DEBG up_ds_listen was notified
Sep 22 23:21:37.880 INFO [0] received activate with gen 1
Sep 22 23:21:37.880 INFO [0] client got ds_active_rx, promote! session 71f2b629-5d8b-450e-858e-3d0aaedbefc8
Sep 22 23:21:37.880 INFO Set check for repair
Sep 22 23:21:37.880 DEBG up_ds_listen process 1002
Sep 22 23:21:37.880 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.880 INFO [1] received activate with gen 1
Sep 22 23:21:37.880 INFO [0] received reconcile message
Sep 22 23:21:37.880 INFO [1] client got ds_active_rx, promote! session 71f2b629-5d8b-450e-858e-3d0aaedbefc8
Sep 22 23:21:37.880 INFO [0] All repairs completed, exit
Sep 22 23:21:37.880 INFO [2] received activate with gen 1
Sep 22 23:21:37.880 INFO [0] Starts cmd_loop
Sep 22 23:21:37.880 INFO [2] client got ds_active_rx, promote! session 71f2b629-5d8b-450e-858e-3d0aaedbefc8
Sep 22 23:21:37.880 INFO [1] received reconcile message
Sep 22 23:21:37.880 INFO [1] All repairs completed, exit
Sep 22 23:21:37.880 INFO UpstairsConnection { upstairs_id: a334e2c5-02d5-47e7-a7d4-82e25a58ab9c, session_id: 71f2b629-5d8b-450e-858e-3d0aaedbefc8, gen: 1 } is now active (read-write)
Sep 22 23:21:37.880 INFO [1] Starts cmd_loop
Sep 22 23:21:37.880 DEBG [2] Read AckReady 1002, : downstairs
Sep 22 23:21:37.880 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.880 INFO UpstairsConnection { upstairs_id: a334e2c5-02d5-47e7-a7d4-82e25a58ab9c, session_id: 71f2b629-5d8b-450e-858e-3d0aaedbefc8, gen: 1 } is now active (read-write)
Sep 22 23:21:37.880 INFO UUID: 4cf53139-503c-4a75-a5fe-71758d156435
Sep 22 23:21:37.880 INFO [2] received reconcile message
Sep 22 23:21:37.880 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.880 INFO [2] All repairs completed, exit
Sep 22 23:21:37.880 INFO [2] Starts cmd_loop
Sep 22 23:21:37.880 INFO UpstairsConnection { upstairs_id: a334e2c5-02d5-47e7-a7d4-82e25a58ab9c, session_id: 71f2b629-5d8b-450e-858e-3d0aaedbefc8, gen: 1 } is now active (read-write)
Sep 22 23:21:37.880 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.880 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.880 INFO Using address: 127.0.0.1:34169, task: main
The guest has finished waiting for activation
Sep 22 23:21:37.880 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.881 INFO [0] downstairs client at 127.0.0.1:64599 has UUID c731e991-4a16-42af-828a-390914aa5a24
Sep 22 23:21:37.881 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c731e991-4a16-42af-828a-390914aa5a24, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.881 INFO a334e2c5-02d5-47e7-a7d4-82e25a58ab9c WaitActive WaitActive WaitActive
Sep 22 23:21:37.881 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.881 INFO [1] downstairs client at 127.0.0.1:62465 has UUID e615c980-e0be-48be-967d-b06ac35abc56
Sep 22 23:21:37.881 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e615c980-e0be-48be-967d-b06ac35abc56, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.881 INFO a334e2c5-02d5-47e7-a7d4-82e25a58ab9c WaitActive WaitActive WaitActive
Sep 22 23:21:37.881 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36924, task: repair
Sep 22 23:21:37.881 INFO [2] downstairs client at 127.0.0.1:56505 has UUID 8e6d336f-2489-4c93-9267-791b6b43d94e
Sep 22 23:21:37.881 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 8e6d336f-2489-4c93-9267-791b6b43d94e, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.881 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36924, task: repair
Sep 22 23:21:37.881 INFO a334e2c5-02d5-47e7-a7d4-82e25a58ab9c WaitActive WaitActive WaitActive
Sep 22 23:21:37.881 INFO listening, local_addr: 127.0.0.1:36924, task: repair
Sep 22 23:21:37.881 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.881 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36924, task: repair
Sep 22 23:21:37.881 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.881 INFO Using repair address: 127.0.0.1:36924, task: main
Sep 22 23:21:37.881 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.881 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.881 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.881 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.881 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.881 INFO Created new region file "/tmp/downstairs-gpKNfyva/region.json"
Sep 22 23:21:37.881 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.882 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.882 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.882 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.882 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.882 INFO [0] a334e2c5-02d5-47e7-a7d4-82e25a58ab9c (71f2b629-5d8b-450e-858e-3d0aaedbefc8) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.882 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.882 WARN [0] new RM replaced this: None
Sep 22 23:21:37.882 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.882 INFO [0] Starts reconcile loop
Sep 22 23:21:37.882 INFO [1] a334e2c5-02d5-47e7-a7d4-82e25a58ab9c (71f2b629-5d8b-450e-858e-3d0aaedbefc8) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.882 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.882 WARN [1] new RM replaced this: None
Sep 22 23:21:37.882 INFO [1] Starts reconcile loop
Sep 22 23:21:37.882 INFO [2] a334e2c5-02d5-47e7-a7d4-82e25a58ab9c (71f2b629-5d8b-450e-858e-3d0aaedbefc8) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.882 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.883 WARN [2] new RM replaced this: None
Sep 22 23:21:37.883 INFO [2] Starts reconcile loop
Sep 22 23:21:37.883 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.883 INFO [0] 127.0.0.1:64599 task reports connection:true
Sep 22 23:21:37.883 INFO a334e2c5-02d5-47e7-a7d4-82e25a58ab9c WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.883 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.883 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.883 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.883 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.883 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.883 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.883 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.883 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.883 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.883 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.883 INFO Max found gen is 1
Sep 22 23:21:37.883 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.883 INFO Next flush: 1
Sep 22 23:21:37.883 INFO All extents match
Sep 22 23:21:37.883 INFO No downstairs repair required
Sep 22 23:21:37.883 INFO No initial repair work was required
Sep 22 23:21:37.883 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.883 INFO a334e2c5-02d5-47e7-a7d4-82e25a58ab9c is now active with session: 71f2b629-5d8b-450e-858e-3d0aaedbefc8
Sep 22 23:21:37.883 DEBG [0] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.883 INFO a334e2c5-02d5-47e7-a7d4-82e25a58ab9c Set Active after no repair
Sep 22 23:21:37.883 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.883 INFO Set check for repair
Sep 22 23:21:37.883 INFO [1] 127.0.0.1:62465 task reports connection:true
Sep 22 23:21:37.883 INFO a334e2c5-02d5-47e7-a7d4-82e25a58ab9c Active Active Active
Sep 22 23:21:37.883 INFO Set check for repair
Sep 22 23:21:37.883 INFO [2] 127.0.0.1:56505 task reports connection:true
Sep 22 23:21:37.883 INFO a334e2c5-02d5-47e7-a7d4-82e25a58ab9c Active Active Active
Sep 22 23:21:37.883 INFO Set check for repair
Sep 22 23:21:37.883 INFO [0] received reconcile message
Sep 22 23:21:37.883 INFO [0] All repairs completed, exit
Sep 22 23:21:37.883 INFO [0] Starts cmd_loop
Sep 22 23:21:37.883 INFO [1] received reconcile message
Sep 22 23:21:37.883 INFO [1] All repairs completed, exit
Sep 22 23:21:37.883 INFO [1] Starts cmd_loop
Sep 22 23:21:37.883 INFO [2] received reconcile message
Sep 22 23:21:37.884 INFO [2] All repairs completed, exit
Sep 22 23:21:37.884 INFO [2] Starts cmd_loop
Sep 22 23:21:37.884 DEBG [2] Read AckReady 1000, : downstairs
The guest has finished waiting for activation
Sep 22 23:21:37.884 DEBG [0] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.884 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.884 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.884 DEBG up_ds_listen was notified
Sep 22 23:21:37.884 DEBG up_ds_listen process 1000
Sep 22 23:21:37.884 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.885 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.885 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.885 DEBG up_ds_listen was notified
Sep 22 23:21:37.885 DEBG up_ds_listen process 1002
Sep 22 23:21:37.885 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.885 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.885 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.886 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.886 INFO Opened existing region file "/tmp/downstairs-gpKNfyva/region.json"
Sep 22 23:21:37.886 INFO Database read version 1
Sep 22 23:21:37.886 INFO Database write version 1
Sep 22 23:21:37.886 DEBG Read      :1000 deps:[] res:true
test test::integration_test_three_layers ... ok
Sep 22 23:21:37.886 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.886 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.886 INFO Created new region file "/tmp/downstairs-0tv4fhPa/region.json"
Sep 22 23:21:37.887 INFO UUID: 814ff37e-28f4-46b8-bc55-126a66e61fb7
Sep 22 23:21:37.887 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.887 DEBG [1] Read AckReady 1000, : downstairs
Sep 22 23:21:37.887 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.887 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.887 INFO Using address: 127.0.0.1:51872, task: main
Sep 22 23:21:37.887 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.887 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52092, task: repair
Sep 22 23:21:37.887 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52092, task: repair
Sep 22 23:21:37.887 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.887 INFO listening, local_addr: 127.0.0.1:52092, task: repair
Sep 22 23:21:37.887 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52092, task: repair
Sep 22 23:21:37.887 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.887 INFO Using repair address: 127.0.0.1:52092, task: main
Sep 22 23:21:37.888 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.888 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.888 DEBG up_ds_listen was notified
Sep 22 23:21:37.888 DEBG up_ds_listen process 1000
Sep 22 23:21:37.888 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.888 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.888 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:37.889 DEBG [2] Read already AckReady 1000, : downstairs
test test::integration_test_two_layers ... ok
Sep 22 23:21:37.890 INFO Upstairs starts
Sep 22 23:21:37.890 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.890 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.890 INFO Crucible stats registered with UUID: 9062e12f-0869-4fd7-b0b2-a9dea3922b3a
Sep 22 23:21:37.890 INFO Crucible 9062e12f-0869-4fd7-b0b2-a9dea3922b3a has session id: 3b2b1948-9af4-49ad-a2ba-c5c3035b4dc4
Sep 22 23:21:37.890 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.890 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.890 INFO Created new region file "/tmp/downstairs-HAffqMEZ/region.json"
Sep 22 23:21:37.890 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.890 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.890 INFO [0] connecting to 127.0.0.1:65200, looper: 0
Sep 22 23:21:37.890 INFO [1] connecting to 127.0.0.1:34169, looper: 1
Sep 22 23:21:37.890 INFO [2] connecting to 127.0.0.1:51872, looper: 2
Sep 22 23:21:37.891 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.891 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.891 INFO Opened existing region file "/tmp/downstairs-0tv4fhPa/region.json"
Sep 22 23:21:37.891 INFO Database read version 1
Sep 22 23:21:37.891 INFO Database write version 1
Sep 22 23:21:37.891 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.891 INFO Flush timeout: 0.5
Sep 22 23:21:37.891 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:37.891 DEBG up_ds_listen was notified
Sep 22 23:21:37.891 DEBG up_ds_listen process 1001
Sep 22 23:21:37.891 INFO [0] 9062e12f-0869-4fd7-b0b2-a9dea3922b3a looper connected, looper: 0
Sep 22 23:21:37.891 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.891 INFO [0] Proc runs for 127.0.0.1:65200 in state New
Sep 22 23:21:37.891 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.891 INFO accepted connection from 127.0.0.1:43906, task: main
Sep 22 23:21:37.891 INFO [1] 9062e12f-0869-4fd7-b0b2-a9dea3922b3a looper connected, looper: 1
Sep 22 23:21:37.891 INFO [1] Proc runs for 127.0.0.1:34169 in state New
Sep 22 23:21:37.891 INFO [2] 9062e12f-0869-4fd7-b0b2-a9dea3922b3a looper connected, looper: 2
Sep 22 23:21:37.891 INFO [2] Proc runs for 127.0.0.1:51872 in state New
Sep 22 23:21:37.891 INFO accepted connection from 127.0.0.1:39800, task: main
Sep 22 23:21:37.892 INFO accepted connection from 127.0.0.1:53303, task: main
Sep 22 23:21:37.892 INFO Connection request from 9062e12f-0869-4fd7-b0b2-a9dea3922b3a with version 4, task: proc
Sep 22 23:21:37.892 INFO upstairs UpstairsConnection { upstairs_id: 9062e12f-0869-4fd7-b0b2-a9dea3922b3a, session_id: 2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.892 INFO Connection request from 9062e12f-0869-4fd7-b0b2-a9dea3922b3a with version 4, task: proc
Sep 22 23:21:37.892 DEBG [0] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.892 INFO upstairs UpstairsConnection { upstairs_id: 9062e12f-0869-4fd7-b0b2-a9dea3922b3a, session_id: 2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.892 DEBG up_ds_listen was notified
Sep 22 23:21:37.892 DEBG up_ds_listen process 1000
Sep 22 23:21:37.892 INFO Connection request from 9062e12f-0869-4fd7-b0b2-a9dea3922b3a with version 4, task: proc
Sep 22 23:21:37.892 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.892 INFO upstairs UpstairsConnection { upstairs_id: 9062e12f-0869-4fd7-b0b2-a9dea3922b3a, session_id: 2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:21:37.892 INFO [0] 9062e12f-0869-4fd7-b0b2-a9dea3922b3a (2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf) New New New ds_transition to WaitActive
Sep 22 23:21:37.892 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.892 INFO UUID: 08d72c33-1ae3-4984-93cb-e5924e7608cb
Sep 22 23:21:37.892 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.892 INFO [1] 9062e12f-0869-4fd7-b0b2-a9dea3922b3a (2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.892 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.892 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.893 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.893 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.893 INFO Using address: 127.0.0.1:50727, task: main
Sep 22 23:21:37.893 INFO [2] 9062e12f-0869-4fd7-b0b2-a9dea3922b3a (2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.893 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.893 INFO 9062e12f-0869-4fd7-b0b2-a9dea3922b3a active request set
Sep 22 23:21:37.893 INFO [0] received activate with gen 1
Sep 22 23:21:37.893 INFO [0] client got ds_active_rx, promote! session 2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf
Sep 22 23:21:37.893 INFO [1] received activate with gen 1
Sep 22 23:21:37.893 INFO [1] client got ds_active_rx, promote! session 2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf
Sep 22 23:21:37.893 DEBG up_ds_listen was notified
Sep 22 23:21:37.893 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.893 DEBG up_ds_listen process 1001
Sep 22 23:21:37.893 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.893 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:21:37.893 INFO [2] received activate with gen 1
Sep 22 23:21:37.893 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.893 INFO [2] client got ds_active_rx, promote! session 2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf
Sep 22 23:21:37.893 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47905, task: repair
Sep 22 23:21:37.893 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47905, task: repair
Sep 22 23:21:37.893 INFO listening, local_addr: 127.0.0.1:47905, task: repair
Sep 22 23:21:37.893 INFO UpstairsConnection { upstairs_id: 9062e12f-0869-4fd7-b0b2-a9dea3922b3a, session_id: 2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf, gen: 1 } is now active (read-write)
Sep 22 23:21:37.893 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:21:37.893 INFO UpstairsConnection { upstairs_id: 9062e12f-0869-4fd7-b0b2-a9dea3922b3a, session_id: 2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf, gen: 1 } is now active (read-write)
Sep 22 23:21:37.893 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.893 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47905, task: repair
Sep 22 23:21:37.893 INFO Opened existing region file "/tmp/downstairs-HAffqMEZ/region.json"
Sep 22 23:21:37.893 INFO Database read version 1
Sep 22 23:21:37.893 INFO Database write version 1
Sep 22 23:21:37.893 INFO Using repair address: 127.0.0.1:47905, task: main
Sep 22 23:21:37.893 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.893 INFO UpstairsConnection { upstairs_id: 9062e12f-0869-4fd7-b0b2-a9dea3922b3a, session_id: 2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf, gen: 1 } is now active (read-write)
Sep 22 23:21:37.894 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:21:37.894 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.894 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:21:37.894 INFO Created new region file "/tmp/downstairs-lpBbEHsY/region.json"
Sep 22 23:21:37.894 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:21:37.894 INFO [0] downstairs client at 127.0.0.1:65200 has UUID f146ddc8-437e-48f6-aa56-398321fa6a78
Sep 22 23:21:37.894 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.894 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f146ddc8-437e-48f6-aa56-398321fa6a78, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.894 INFO 9062e12f-0869-4fd7-b0b2-a9dea3922b3a WaitActive WaitActive WaitActive
Sep 22 23:21:37.894 INFO [1] downstairs client at 127.0.0.1:34169 has UUID 4cf53139-503c-4a75-a5fe-71758d156435
Sep 22 23:21:37.894 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4cf53139-503c-4a75-a5fe-71758d156435, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.894 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.894 INFO 9062e12f-0869-4fd7-b0b2-a9dea3922b3a WaitActive WaitActive WaitActive
Sep 22 23:21:37.894 DEBG up_ds_listen was notified
Sep 22 23:21:37.894 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.894 INFO [2] downstairs client at 127.0.0.1:51872 has UUID 814ff37e-28f4-46b8-bc55-126a66e61fb7
Sep 22 23:21:37.894 DEBG up_ds_listen process 1001
Sep 22 23:21:37.894 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 814ff37e-28f4-46b8-bc55-126a66e61fb7, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.894 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.894 INFO 9062e12f-0869-4fd7-b0b2-a9dea3922b3a WaitActive WaitActive WaitActive
Sep 22 23:21:37.894 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:21:37.894 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.894 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.894 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.894 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.895 INFO Created new region file "/tmp/downstairs-n0ieH1oY/region.json"
Sep 22 23:21:37.895 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.895 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.895 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.895 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.896 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.896 INFO UUID: d300473c-ae2a-4754-900f-58509b1b9961
Sep 22 23:21:37.896 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.896 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.896 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.896 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.896 INFO Using address: 127.0.0.1:33968, task: main
Sep 22 23:21:37.896 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:37.896 INFO [0] 9062e12f-0869-4fd7-b0b2-a9dea3922b3a (2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.896 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.896 WARN [0] new RM replaced this: None
Sep 22 23:21:37.896 INFO [0] Starts reconcile loop
Sep 22 23:21:37.896 INFO [1] 9062e12f-0869-4fd7-b0b2-a9dea3922b3a (2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.896 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.896 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:21:37.896 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.896 WARN [1] new RM replaced this: None
Sep 22 23:21:37.896 INFO [1] Starts reconcile loop
Sep 22 23:21:37.896 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:34919, task: repair
Sep 22 23:21:37.896 INFO [2] 9062e12f-0869-4fd7-b0b2-a9dea3922b3a (2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.896 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.896 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:34919, task: repair
Sep 22 23:21:37.896 WARN [2] new RM replaced this: None
Sep 22 23:21:37.896 INFO [2] Starts reconcile loop
Sep 22 23:21:37.896 INFO listening, local_addr: 127.0.0.1:34919, task: repair
Sep 22 23:21:37.896 INFO [0] 127.0.0.1:65200 task reports connection:true
Sep 22 23:21:37.896 INFO 9062e12f-0869-4fd7-b0b2-a9dea3922b3a WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.896 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.896 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.897 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.897 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:34919, task: repair
Sep 22 23:21:37.897 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.897 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.897 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.897 INFO Using repair address: 127.0.0.1:34919, task: main
Sep 22 23:21:37.897 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.897 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.897 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.897 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.897 INFO Max found gen is 1
Sep 22 23:21:37.897 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.897 INFO Next flush: 1
Sep 22 23:21:37.897 INFO All extents match
Sep 22 23:21:37.897 INFO No downstairs repair required
Sep 22 23:21:37.897 INFO No initial repair work was required
Sep 22 23:21:37.897 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.897 INFO 9062e12f-0869-4fd7-b0b2-a9dea3922b3a is now active with session: 2a9cbe7b-00ff-4e9a-a06a-da8422b4d8bf
Sep 22 23:21:37.897 INFO 9062e12f-0869-4fd7-b0b2-a9dea3922b3a Set Active after no repair
Sep 22 23:21:37.897 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.897 INFO Set check for repair
Sep 22 23:21:37.897 INFO [1] 127.0.0.1:34169 task reports connection:true
Sep 22 23:21:37.897 INFO 9062e12f-0869-4fd7-b0b2-a9dea3922b3a Active Active Active
Sep 22 23:21:37.897 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.897 INFO Set check for repair
Sep 22 23:21:37.897 INFO [2] 127.0.0.1:51872 task reports connection:true
Sep 22 23:21:37.897 INFO 9062e12f-0869-4fd7-b0b2-a9dea3922b3a Active Active Active
Sep 22 23:21:37.897 INFO Created new region file "/tmp/downstairs-KmXVVDA0/region.json"
Sep 22 23:21:37.897 INFO Set check for repair
Sep 22 23:21:37.897 INFO [0] received reconcile message
Sep 22 23:21:37.897 INFO [0] All repairs completed, exit
Sep 22 23:21:37.897 INFO [0] Starts cmd_loop
Sep 22 23:21:37.897 INFO [1] received reconcile message
Sep 22 23:21:37.897 INFO [1] All repairs completed, exit
Sep 22 23:21:37.897 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.897 INFO [1] Starts cmd_loop
Sep 22 23:21:37.897 INFO [2] received reconcile message
Sep 22 23:21:37.897 INFO [2] All repairs completed, exit
Sep 22 23:21:37.897 INFO [2] Starts cmd_loop
Sep 22 23:21:37.897 DEBG [2] Read AckReady 1002, : downstairs
The guest has finished waiting for activation
Sep 22 23:21:37.898 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.898 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.898 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.898 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.898 INFO Opened existing region file "/tmp/downstairs-lpBbEHsY/region.json"
Sep 22 23:21:37.898 INFO Database read version 1
Sep 22 23:21:37.898 INFO Database write version 1
Sep 22 23:21:37.899 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.899 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.899 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.900 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.900 INFO Opened existing region file "/tmp/downstairs-n0ieH1oY/region.json"
Sep 22 23:21:37.900 INFO Database read version 1
Sep 22 23:21:37.900 INFO Database write version 1
Sep 22 23:21:37.900 DEBG [0] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.901 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.901 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.901 INFO UUID: f233f1e5-b968-4e9a-959c-120f95afdd47
Sep 22 23:21:37.901 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.901 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.901 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.901 INFO Using address: 127.0.0.1:50019, task: main
Sep 22 23:21:37.901 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.901 DEBG up_ds_listen was notified
Sep 22 23:21:37.901 DEBG up_ds_listen process 1000
Sep 22 23:21:37.901 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.901 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.901 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38365, task: repair
Sep 22 23:21:37.901 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38365, task: repair
Sep 22 23:21:37.902 INFO listening, local_addr: 127.0.0.1:38365, task: repair
Sep 22 23:21:37.902 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.902 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38365, task: repair
Sep 22 23:21:37.902 INFO Using repair address: 127.0.0.1:38365, task: main
Sep 22 23:21:37.902 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.902 INFO UUID: e1b44f76-91da-4986-a06d-4f047931e7eb
Sep 22 23:21:37.902 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.902 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.902 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.902 INFO Using address: 127.0.0.1:51174, task: main
Sep 22 23:21:37.902 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.902 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.902 INFO Opened existing region file "/tmp/downstairs-KmXVVDA0/region.json"
Sep 22 23:21:37.902 INFO Database read version 1
Sep 22 23:21:37.902 INFO Database write version 1
Sep 22 23:21:37.902 INFO Created new region file "/tmp/downstairs-hTrUSJqe/region.json"
Sep 22 23:21:37.902 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:21:37.902 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.902 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40366, task: repair
Sep 22 23:21:37.902 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40366, task: repair
Sep 22 23:21:37.902 INFO listening, local_addr: 127.0.0.1:40366, task: repair
Sep 22 23:21:37.903 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40366, task: repair
Sep 22 23:21:37.903 INFO Using repair address: 127.0.0.1:40366, task: main
Sep 22 23:21:37.903 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.903 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.903 DEBG up_ds_listen was notified
Sep 22 23:21:37.903 DEBG up_ds_listen process 1002
Sep 22 23:21:37.903 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.903 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.903 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.903 INFO Created new region file "/tmp/downstairs-QR6s6qBv/region.json"
Sep 22 23:21:37.904 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.905 INFO UUID: 84d77caf-cd99-479d-b329-65e270f8e6d5
Sep 22 23:21:37.905 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.905 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.905 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.905 INFO Using address: 127.0.0.1:40280, task: main
Sep 22 23:21:37.905 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.905 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:41802, task: repair
Sep 22 23:21:37.905 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:41802, task: repair
Sep 22 23:21:37.905 INFO listening, local_addr: 127.0.0.1:41802, task: repair
Sep 22 23:21:37.906 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:41802, task: repair
Sep 22 23:21:37.906 INFO Using repair address: 127.0.0.1:41802, task: main
Sep 22 23:21:37.906 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.906 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:37.906 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.906 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.906 DEBG up_ds_listen was notified
Sep 22 23:21:37.906 DEBG up_ds_listen process 1002
Sep 22 23:21:37.906 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.906 INFO Created new region file "/tmp/downstairs-sbyjS0xN/region.json"
Sep 22 23:21:37.906 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.907 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.907 INFO Opened existing region file "/tmp/downstairs-hTrUSJqe/region.json"
Sep 22 23:21:37.907 INFO Database read version 1
Sep 22 23:21:37.907 INFO Database write version 1
test test::integration_test_two_layers_parent_smaller ... ok
Sep 22 23:21:37.910 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.910 INFO Created new region file "/tmp/downstairs-E6VpKmdW/region.json"
Sep 22 23:21:37.910 INFO UUID: 5ad4cc94-cc5d-4745-8f62-94ec4413158f
Sep 22 23:21:37.910 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.910 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.910 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.910 INFO Using address: 127.0.0.1:35408, task: main
Sep 22 23:21:37.911 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.911 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.911 INFO Opened existing region file "/tmp/downstairs-QR6s6qBv/region.json"
Sep 22 23:21:37.911 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:54039, task: repair
Sep 22 23:21:37.911 INFO Database read version 1
Sep 22 23:21:37.911 INFO Database write version 1
Sep 22 23:21:37.911 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:54039, task: repair
Sep 22 23:21:37.911 INFO listening, local_addr: 127.0.0.1:54039, task: repair
test test::integration_test_two_layers_parent_smaller_unwritten ... ok
Sep 22 23:21:37.911 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:54039, task: repair
Sep 22 23:21:37.911 INFO Using repair address: 127.0.0.1:54039, task: main
Sep 22 23:21:37.911 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.911 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.911 INFO Opened existing region file "/tmp/downstairs-sbyjS0xN/region.json"
Sep 22 23:21:37.911 INFO Database read version 1
Sep 22 23:21:37.911 INFO Database write version 1
Sep 22 23:21:37.911 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.912 INFO Created new region file "/tmp/downstairs-HlXYD8jF/region.json"
{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.912401547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.912466678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.912490532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"The guest has requested activation
Crucible stats registered with UUID: 5890b7ae-9696-4b48-a320-1c616f83f794","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.912523208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible 5890b7ae-9696-4b48-a320-1c616f83f794 has session id: 697e29ca-b1d6-4e8f-9151-6ed6ab5a7f62","v":0,"name":"crucible","level":30,"time":"Sep 22 23:21:37.912 INFO listening on 127.0.0.1:0, task: main
2023-09-22T23:21:37.91256345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.912 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.912 INFO listening on 127.0.0.1:0, task: main
{"msg":"[0] connecting to 127.0.0.1:50727","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.912805993Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
{"msg":"[1] connecting to 127.0.0.1:50019","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.912945337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1"}
Sep 22 23:21:37.913 INFO UUID: fb1574ce-cb6c-41db-a1fc-e45b9d42517f
{"msg":"Sep 22 23:21:37.913 INFO Blocks per extent:5 Total Extents: 2
[2] connecting to 127.0.0.1:35408","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91311039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2"}
Sep 22 23:21:37.913 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
{"msg":"up_listen starts","v":0,"name":"crucible","level":30Sep 22 23:21:37.913 INFO Upstairs <-> Downstairs Message Version: 4, task: main
,"time":"2023-09-22T23:21:37.913197938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","Sep 22 23:21:37.913 INFO Using address: 127.0.0.1:59981, task: main
pid":4769,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.913249193Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91328745Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"5890b7ae-9696-4b48-a320-1c616f83f794 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.913321841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.913 INFO accepted connection from 127.0.0.1:33366, task: main
Sep 22 23:21:37.913 INFO accepted connection from 127.0.0.1:50854, task: main
Sep 22 23:21:37.913 INFO Repair listens on 127.0.0.1:0, task: repair
{"msg":"[0] 5890b7ae-9696-4b48-a320-1c616f83f794 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91360681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769Sep 22 23:21:37.913 DEBG up_ds_listen was notified
Sep 22 23:21:37.913 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37110, task: repair
,"looper":"0"}
{"msg":"[0] Proc runs for 127.0.0.1:50727 in state New"Sep 22 23:21:37.913 DEBG up_ds_listen process 1001
,"v":Sep 22 23:21:37.913 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37110, task: repair
0,"name":"crucible","level":30Sep 22 23:21:37.913 DEBG [A] ack job 1001:2, : downstairs
,"time":"2023-09-22T23:21:37.91369746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.913 INFO listening, local_addr: 127.0.0.1:37110, task: repair
{"msg":"[2] 5890b7ae-9696-4b48-a320-1c616f83f794 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.913746902Z"Sep 22 23:21:37.913 DEBG up_ds_listen checked 1 jobs, back to waiting
,"hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2"}
{"msg":"[2] Proc runs for 127.0.0.1:35408 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.913797532Z","hostnameSep 22 23:21:37.913 INFO UUID: 1104ac55-f7a8-4cef-b93e-5cab8f0588d7
":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Sep 22 23:21:37.913 INFO Blocks per extent:5 Total Extents: 2
[1] 5890b7ae-9696-4b48-a320-1c616f83f794 looper connected","v":0,"name":"crucible","level":30Sep 22 23:21:37.913 INFO accepted connection from 127.0.0.1:36315, task: main
,"time":"2023-09-22T23:21:37.91384397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1"}
{"msg":"[1] Proc runs for 127.0.0.1:50019 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91388497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.913 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37110, task: repair
Sep 22 23:21:37.913 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.913 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.913 INFO Using repair address: 127.0.0.1:37110, task: main
Sep 22 23:21:37.913 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.913 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.913 INFO Using address: 127.0.0.1:36377, task: main
Sep 22 23:21:37.913 INFO Opened existing region file "/tmp/downstairs-E6VpKmdW/region.json"
Sep 22 23:21:37.913 INFO Database read version 1
Sep 22 23:21:37.913 INFO Database write version 1
Sep 22 23:21:37.914 INFO Connection request from 5890b7ae-9696-4b48-a320-1c616f83f794 with version 4, task: proc
Sep 22 23:21:37.914 INFO upstairs UpstairsConnection { upstairs_id: 5890b7ae-9696-4b48-a320-1c616f83f794, session_id: 2ccf0845-edb5-4c6d-8f10-f9f1a2498395, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.914 INFO Connection request from 5890b7ae-9696-4b48-a320-1c616f83f794 with version 4, task: proc
Sep 22 23:21:37.914 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.914 INFO upstairs UpstairsConnection { upstairs_id: 5890b7ae-9696-4b48-a320-1c616f83f794, session_id: 2ccf0845-edb5-4c6d-8f10-f9f1a2498395, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.914 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57088, task: repair
Sep 22 23:21:37.914 INFO Connection request from 5890b7ae-9696-4b48-a320-1c616f83f794 with version 4, task: proc
Sep 22 23:21:37.914 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57088, task: repair
Sep 22 23:21:37.914 INFO upstairs UpstairsConnection { upstairs_id: 5890b7ae-9696-4b48-a320-1c616f83f794, session_id: 2ccf0845-edb5-4c6d-8f10-f9f1a2498395, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.914 INFO listening, local_addr: 127.0.0.1:57088, task: repair
Sep 22 23:21:37.914 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57088, task: repair
{"msg":"[0] 5890b7ae-9696-4b48-a320-1c616f83f794 (2ccf0845-edb5-4c6d-8f10-f9f1a2498395) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":Sep 22 23:21:37.914 INFO Using repair address: 127.0.0.1:57088, task: main
30Sep 22 23:21:37.914 INFO No SSL acceptor configured, task: main
,"time":"2023-09-22T23:21:37.914545378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"Sep 22 23:21:37.914 DEBG IO Read  1002 has deps [JobId(1001)]
time":"2023-09-22T23:21:37.914593007Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] client is_active_req TRUE, promote! session 2ccf0845-edb5-4c6d-8f10-f9f1a2498395","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.914643732Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 5890b7ae-9696-4b48-a320-1c616f83f794 (2ccf0845-edb5-4c6d-8f10-f9f1a2498395) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.914679733Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.914705006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] client is_active_req TRUE, promote! session 2ccf0845-edb5-4c6d-8f10-f9f1a2498395","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91473979Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 5890b7ae-9696-4b48-a320-1c616f83f794 (2ccf0845-edb5-4c6d-8f10-f9f1a2498395) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91477636Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.914808134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{Sep 22 23:21:37.914 INFO current number of open files limit 65536 is already the maximum
"msg":"[1] client is_active_req TRUE, promote! session 2ccf0845-edb5-4c6d-8f10-f9f1a2498395","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.914847028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.914 INFO Created new region file "/tmp/downstairs-RaXizMDx/region.json"
Sep 22 23:21:37.915 DEBG Read      :1002 deps:[JobId(1001)] res:true
{"msg":"[0] downstairs client at 127.0.0.1:50727 has UUID 08d72c33-1ae3-4984-93cb-e5924e7608cb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.915549426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 08d72c33-1ae3-4984-93cb-e5924e7608cb, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.915606595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"5890b7ae-9696-4b48-a320-1c616f83f794 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.915643695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] downstairs client at 127.0.0.1:35408 has UUID 5ad4cc94-cc5d-4745-8f62-94ec4413158f","v":0,"name":"crucible","level":30Sep 22 23:21:37.915 INFO Upstairs starts
,"time":"2023-09-22T23:21:37.915684256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 5ad4cc94-cc5d-4745-8f62-94ec4413158f, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30Sep 22 23:21:37.915 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
,"time":"2023-09-22T23:21:37.915729142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{Sep 22 23:21:37.915 INFO Upstairs <-> Downstairs Message Version: 4
"msg":"5890b7ae-9696-4b48-a320-1c616f83f794 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91576738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.915 INFO Crucible stats registered with UUID: d15c167b-c380-46a4-a953-864fc02495ed
Sep 22 23:21:37.915 INFO Current flush_numbers [0..12]: [0, 0]
{"msg":"[1] downstairs client at 127.0.0.1:50019 has UUID f233f1e5-b968-4e9a-959c-120f95afdd47","v":0,"name":"crucible","level":30,"time":"Sep 22 23:21:37.915 INFO Crucible d15c167b-c380-46a4-a953-864fc02495ed has session id: d5a13612-c3c3-4fc0-8a00-21ce9f97f75d
2023-09-22T23:21:37.915816005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f233f1e5-b968-4e9a-959c-120f95afdd47, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30Sep 22 23:21:37.915 DEBG Read      :1002 deps:[JobId(1001)] res:true
,"time":"2023-09-22T23:21:37.915862975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"5890b7ae-9696-4b48-a320-1c616f83f794 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.915906849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.915 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.915 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.916 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.916 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.916 INFO [0] connecting to 127.0.0.1:33968, looper: 0
Sep 22 23:21:37.916 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.916 INFO [1] connecting to 127.0.0.1:40280, looper: 1
Sep 22 23:21:37.916 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:37.916 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.916 INFO [2] connecting to 127.0.0.1:36377, looper: 2
Sep 22 23:21:37.916 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.916 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.916 INFO Flush timeout: 0.5
Sep 22 23:21:37.916 INFO UUID: 80c20ebc-7f9f-4109-87bd-ff9af4ac5180
Sep 22 23:21:37.916 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.916 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.916 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.916 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.916 INFO Using address: 127.0.0.1:60187, task: main
Sep 22 23:21:37.916 INFO accepted connection from 127.0.0.1:40391, task: main
Sep 22 23:21:37.916 INFO accepted connection from 127.0.0.1:50010, task: main
Sep 22 23:21:37.916 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.916 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.916 INFO accepted connection from 127.0.0.1:57192, task: main
Sep 22 23:21:37.916 INFO Opened existing region file "/tmp/downstairs-HlXYD8jF/region.json"
Sep 22 23:21:37.916 INFO Database read version 1
Sep 22 23:21:37.916 INFO Database write version 1
Sep 22 23:21:37.916 INFO [0] d15c167b-c380-46a4-a953-864fc02495ed looper connected, looper: 0
Sep 22 23:21:37.916 INFO [0] Proc runs for 127.0.0.1:33968 in state New
Sep 22 23:21:37.917 INFO [1] d15c167b-c380-46a4-a953-864fc02495ed looper connected, looper: 1
Sep 22 23:21:37.917 INFO Repair listens on 127.0.0.1:0, task: repair
{"msg":"Sep 22 23:21:37.917 INFO [1] Proc runs for 127.0.0.1:40280 in state New
[0] 5890b7ae-9696-4b48-a320-1c616f83f794 (2ccf0845-edb5-4c6d-8f10-f9f1a2498395) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917122802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30Sep 22 23:21:37.917 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50831, task: repair
,"Sep 22 23:21:37.917 INFO [2] d15c167b-c380-46a4-a953-864fc02495ed looper connected, looper: 2
time":"2023-09-22T23:21:37.917174668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] new RM replaced this: NoneSep 22 23:21:37.917 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50831, task: repair
","v":0,"name":"crucible","level":40Sep 22 23:21:37.917 INFO [2] Proc runs for 127.0.0.1:36377 in state New
,"time":"2023-09-22T23:21:37.917224605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30Sep 22 23:21:37.917 INFO listening, local_addr: 127.0.0.1:50831, task: repair
,"time":"2023-09-22T23:21:37.917258341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 5890b7ae-9696-4b48-a320-1c616f83f794 (2ccf0845-edb5-4c6d-8f10-f9f1a2498395) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917297199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91733541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:37.917371196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917403853Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 5890b7ae-9696-4b48-a320-1c616f83f794 (2ccf0845-edb5-4c6d-8f10-f9f1a2498395) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"Sep 22 23:21:37.917 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50831, task: repair
crucible","level":30,"time":"2023-09-22T23:21:37.91744913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0Sep 22 23:21:37.917 INFO Using repair address: 127.0.0.1:50831, task: main
,"name":"crucible","level":30Sep 22 23:21:37.917 INFO No SSL acceptor configured, task: main
,"time":"2023-09-22T23:21:37.917491801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:21:37.917533836Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917570987Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] 127.0.0.1:50727 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917608525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"5890b7ae-9696-4b48-a320-1c616f83f794 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917646372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917693118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91774606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917783606Z","The guest has finished waiting for activation
hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.9178265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917862115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917894596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917923287Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917952243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.917980685Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918009407Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30Sep 22 23:21:37.918 INFO current number of open files limit 65536 is already the maximum
,"time":"2023-09-22T23:21:37.918042411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918081088Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918116808Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918151831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.918 INFO Created new region file "/tmp/downstairs-gtLUOPnz/region.json"
{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918188365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918226516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"5890b7ae-9696-4b48-a320-1c616f83f794 is now active with session: 2ccf0845-edb5-4c6d-8f10-f9f1a2498395","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918266615Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"5890b7ae-9696-4b48-a320-1c616f83f794 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918303185Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918336217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918366854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 127.0.0.1:35408 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91840265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"5890b7ae-9696-4b48-a320-1c616f83f794 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918443508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918477749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 127.0.0.1:50019 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918513514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"5890b7ae-9696-4b48-a320-1c616f83f794 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918550862Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918587489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918622387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918656121Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918689684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918762864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918800276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918836196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.91887184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:21:37.918905849Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:37.919 INFO UUID: b708efc7-8172-4645-a89a-1dd5a6e86ff4
Sep 22 23:21:37.919 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.919 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.919 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.919 INFO Using address: 127.0.0.1:41387, task: main
Sep 22 23:21:37.920 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.920 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48043, task: repair
Sep 22 23:21:37.920 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48043, task: repair
Sep 22 23:21:37.920 INFO listening, local_addr: 127.0.0.1:48043, task: repair
Sep 22 23:21:37.920 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48043, task: repair
Sep 22 23:21:37.920 INFO Using repair address: 127.0.0.1:48043, task: main
Sep 22 23:21:37.920 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.920 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:21:37.920 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.920 INFO Opened existing region file "/tmp/downstairs-RaXizMDx/region.json"
Sep 22 23:21:37.921 INFO Database read version 1
Sep 22 23:21:37.921 INFO Database write version 1
Sep 22 23:21:37.922 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.922 INFO Created new region file "/tmp/downstairs-vbQXcM4a/region.json"
Sep 22 23:21:37.922 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.922 INFO UUID: ab64c062-c03c-4bae-8346-acfa3ee4c8e7
Sep 22 23:21:37.922 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.922 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.923 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.923 INFO Using address: 127.0.0.1:40994, task: main
Sep 22 23:21:37.923 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.923 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.923 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33303, task: repair
Sep 22 23:21:37.923 INFO Opened existing region file "/tmp/downstairs-gtLUOPnz/region.json"
Sep 22 23:21:37.923 INFO Database read version 1
Sep 22 23:21:37.923 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33303, task: repair
Sep 22 23:21:37.923 INFO Database write version 1
Sep 22 23:21:37.923 INFO listening, local_addr: 127.0.0.1:33303, task: repair
Sep 22 23:21:37.923 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33303, task: repair
Sep 22 23:21:37.923 INFO Using repair address: 127.0.0.1:33303, task: main
Sep 22 23:21:37.923 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.924 INFO Upstairs starts
Sep 22 23:21:37.924 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.924 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.924 INFO Crucible stats registered with UUID: 517f6bf3-86c2-4511-bc4e-8840c2dfa85c
Sep 22 23:21:37.924 INFO Crucible 517f6bf3-86c2-4511-bc4e-8840c2dfa85c has session id: 60473939-7206-48a5-b1d3-23c365304810
Sep 22 23:21:37.924 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.924 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.925 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.925 INFO [0] connecting to 127.0.0.1:51174, looper: 0
Sep 22 23:21:37.925 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:21:37.925 INFO [1] connecting to 127.0.0.1:59981, looper: 1
Sep 22 23:21:37.925 INFO [2] connecting to 127.0.0.1:40994, looper: 2
Sep 22 23:21:37.925 DEBG up_ds_listen was notified
Sep 22 23:21:37.925 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.925 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.925 INFO Flush timeout: 0.5
Sep 22 23:21:37.925 DEBG up_ds_listen process 1002
Sep 22 23:21:37.925 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.925 INFO accepted connection from 127.0.0.1:56403, task: main
Sep 22 23:21:37.925 INFO accepted connection from 127.0.0.1:57221, task: main
Sep 22 23:21:37.925 INFO accepted connection from 127.0.0.1:38511, task: main
Sep 22 23:21:37.925 INFO [0] 517f6bf3-86c2-4511-bc4e-8840c2dfa85c looper connected, looper: 0
Sep 22 23:21:37.925 INFO [0] Proc runs for 127.0.0.1:51174 in state New
Sep 22 23:21:37.925 INFO [1] 517f6bf3-86c2-4511-bc4e-8840c2dfa85c looper connected, looper: 1
Sep 22 23:21:37.925 INFO [1] Proc runs for 127.0.0.1:59981 in state New
Sep 22 23:21:37.925 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.925 INFO [2] 517f6bf3-86c2-4511-bc4e-8840c2dfa85c looper connected, looper: 2
Sep 22 23:21:37.925 INFO [2] Proc runs for 127.0.0.1:40994 in state New
Sep 22 23:21:37.926 INFO Upstairs starts
Sep 22 23:21:37.926 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.926 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.926 INFO Crucible stats registered with UUID: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5
Sep 22 23:21:37.926 INFO Crucible 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 has session id: 8053272a-0e59-472a-830e-9cb2148ea8b6
Sep 22 23:21:37.926 INFO Connection request from 517f6bf3-86c2-4511-bc4e-8840c2dfa85c with version 4, task: proc
Sep 22 23:21:37.926 INFO UUID: 6b1a9c06-348a-42ce-a59e-2b95a9617acf
Sep 22 23:21:37.926 INFO upstairs UpstairsConnection { upstairs_id: 517f6bf3-86c2-4511-bc4e-8840c2dfa85c, session_id: 9a97b4b8-4148-45e4-bbb1-302708e60500, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:37.926 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.926 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.926 INFO Connection request from 517f6bf3-86c2-4511-bc4e-8840c2dfa85c with version 4, task: proc
Sep 22 23:21:37.926 INFO Opened existing region file "/tmp/downstairs-vbQXcM4a/region.json"
Sep 22 23:21:37.926 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.926 INFO Database read version 1
Sep 22 23:21:37.926 INFO upstairs UpstairsConnection { upstairs_id: 517f6bf3-86c2-4511-bc4e-8840c2dfa85c, session_id: 9a97b4b8-4148-45e4-bbb1-302708e60500, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:37.926 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.926 INFO Database write version 1
Sep 22 23:21:37.926 INFO Using address: 127.0.0.1:64895, task: main
Sep 22 23:21:37.926 INFO Connection request from 517f6bf3-86c2-4511-bc4e-8840c2dfa85c with version 4, task: proc
Sep 22 23:21:37.926 INFO upstairs UpstairsConnection { upstairs_id: 517f6bf3-86c2-4511-bc4e-8840c2dfa85c, session_id: 9a97b4b8-4148-45e4-bbb1-302708e60500, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:37.926 INFO [0] connecting to 127.0.0.1:48404, looper: 0
Sep 22 23:21:37.926 INFO [1] connecting to 127.0.0.1:54505, looper: 1
Sep 22 23:21:37.927 INFO [2] connecting to 127.0.0.1:38410, looper: 2
Sep 22 23:21:37.927 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.927 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.927 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.927 INFO Flush timeout: 0.5
Sep 22 23:21:37.927 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:41794, task: repair
Sep 22 23:21:37.927 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:41794, task: repair
Sep 22 23:21:37.927 INFO listening, local_addr: 127.0.0.1:41794, task: repair
Sep 22 23:21:37.927 INFO accepted connection from 127.0.0.1:64681, task: main
Sep 22 23:21:37.927 INFO accepted connection from 127.0.0.1:48744, task: main
Sep 22 23:21:37.927 INFO accepted connection from 127.0.0.1:44304, task: main
Sep 22 23:21:37.927 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:41794, task: repair
Sep 22 23:21:37.927 INFO Using repair address: 127.0.0.1:41794, task: main
Sep 22 23:21:37.927 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.927 INFO [0] 517f6bf3-86c2-4511-bc4e-8840c2dfa85c (9a97b4b8-4148-45e4-bbb1-302708e60500) New New New ds_transition to WaitActive
Sep 22 23:21:37.927 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.927 INFO [1] 517f6bf3-86c2-4511-bc4e-8840c2dfa85c (9a97b4b8-4148-45e4-bbb1-302708e60500) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.927 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.927 INFO [2] 517f6bf3-86c2-4511-bc4e-8840c2dfa85c (9a97b4b8-4148-45e4-bbb1-302708e60500) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.927 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.927 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 looper connected, looper: 0
Sep 22 23:21:37.927 INFO [0] Proc runs for 127.0.0.1:48404 in state New
Sep 22 23:21:37.927 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 looper connected, looper: 1
Sep 22 23:21:37.927 INFO [1] Proc runs for 127.0.0.1:54505 in state New
Sep 22 23:21:37.927 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 looper connected, looper: 2
Sep 22 23:21:37.927 INFO [2] Proc runs for 127.0.0.1:38410 in state New
Sep 22 23:21:37.927 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.928 INFO Created new region file "/tmp/downstairs-TcKuMYwD/region.json"
Sep 22 23:21:37.928 INFO Connection request from 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 with version 4, task: proc
Sep 22 23:21:37.928 INFO upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 2e580a9f-7e54-44b6-83db-09df314d2c01, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:37.928 INFO Connection request from 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 with version 4, task: proc
Sep 22 23:21:37.928 INFO upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 2e580a9f-7e54-44b6-83db-09df314d2c01, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:37.928 INFO Connection request from 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 with version 4, task: proc
Sep 22 23:21:37.928 INFO upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 2e580a9f-7e54-44b6-83db-09df314d2c01, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:37.928 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (2e580a9f-7e54-44b6-83db-09df314d2c01) New New New ds_transition to WaitActive
Sep 22 23:21:37.928 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.928 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (2e580a9f-7e54-44b6-83db-09df314d2c01) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.928 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.928 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (2e580a9f-7e54-44b6-83db-09df314d2c01) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.928 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:21:37.928 INFO 517f6bf3-86c2-4511-bc4e-8840c2dfa85c active request set
Sep 22 23:21:37.929 INFO [0] received activate with gen 3
Sep 22 23:21:37.929 INFO [0] client got ds_active_rx, promote! session 9a97b4b8-4148-45e4-bbb1-302708e60500
Sep 22 23:21:37.929 INFO [1] received activate with gen 3
Sep 22 23:21:37.929 INFO [1] client got ds_active_rx, promote! session 9a97b4b8-4148-45e4-bbb1-302708e60500
Sep 22 23:21:37.929 INFO [2] received activate with gen 3
Sep 22 23:21:37.929 INFO [2] client got ds_active_rx, promote! session 9a97b4b8-4148-45e4-bbb1-302708e60500
Sep 22 23:21:37.929 INFO UpstairsConnection { upstairs_id: 517f6bf3-86c2-4511-bc4e-8840c2dfa85c, session_id: 9a97b4b8-4148-45e4-bbb1-302708e60500, gen: 3 } is now active (read-write)
Sep 22 23:21:37.929 INFO UpstairsConnection { upstairs_id: 517f6bf3-86c2-4511-bc4e-8840c2dfa85c, session_id: 9a97b4b8-4148-45e4-bbb1-302708e60500, gen: 3 } is now active (read-write)
Sep 22 23:21:37.929 INFO UpstairsConnection { upstairs_id: 517f6bf3-86c2-4511-bc4e-8840c2dfa85c, session_id: 9a97b4b8-4148-45e4-bbb1-302708e60500, gen: 3 } is now active (read-write)
Sep 22 23:21:37.929 INFO UUID: 2dedfda5-97c1-427a-8b74-5c08fb2c233b
Sep 22 23:21:37.929 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.929 INFO [0] downstairs client at 127.0.0.1:51174 has UUID e1b44f76-91da-4986-a06d-4f047931e7eb
Sep 22 23:21:37.929 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e1b44f76-91da-4986-a06d-4f047931e7eb, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.929 INFO 517f6bf3-86c2-4511-bc4e-8840c2dfa85c WaitActive WaitActive WaitActive
Sep 22 23:21:37.929 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.929 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.929 INFO Using address: 127.0.0.1:58081, task: main
Sep 22 23:21:37.929 INFO [1] downstairs client at 127.0.0.1:59981 has UUID fb1574ce-cb6c-41db-a1fc-e45b9d42517f
Sep 22 23:21:37.930 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: fb1574ce-cb6c-41db-a1fc-e45b9d42517f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.930 INFO 517f6bf3-86c2-4511-bc4e-8840c2dfa85c WaitActive WaitActive WaitActive
Sep 22 23:21:37.930 INFO [2] downstairs client at 127.0.0.1:40994 has UUID ab64c062-c03c-4bae-8346-acfa3ee4c8e7
Sep 22 23:21:37.930 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ab64c062-c03c-4bae-8346-acfa3ee4c8e7, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.930 INFO 517f6bf3-86c2-4511-bc4e-8840c2dfa85c WaitActive WaitActive WaitActive
Sep 22 23:21:37.930 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.930 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61379, task: repair
Sep 22 23:21:37.930 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61379, task: repair
Sep 22 23:21:37.930 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.930 INFO listening, local_addr: 127.0.0.1:61379, task: repair
Sep 22 23:21:37.930 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.930 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.930 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61379, task: repair
Sep 22 23:21:37.930 INFO Using repair address: 127.0.0.1:61379, task: main
Sep 22 23:21:37.930 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.931 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.931 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.931 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.931 INFO Created new region file "/tmp/downstairs-XLPx1jrJ/region.json"
Sep 22 23:21:37.931 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.931 INFO [0] 517f6bf3-86c2-4511-bc4e-8840c2dfa85c (9a97b4b8-4148-45e4-bbb1-302708e60500) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.931 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.931 WARN [0] new RM replaced this: None
Sep 22 23:21:37.931 INFO [0] Starts reconcile loop
Sep 22 23:21:37.931 INFO [1] 517f6bf3-86c2-4511-bc4e-8840c2dfa85c (9a97b4b8-4148-45e4-bbb1-302708e60500) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.931 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.931 WARN [1] new RM replaced this: None
Sep 22 23:21:37.931 INFO [1] Starts reconcile loop
Sep 22 23:21:37.931 INFO [2] 517f6bf3-86c2-4511-bc4e-8840c2dfa85c (9a97b4b8-4148-45e4-bbb1-302708e60500) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.931 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.931 WARN [2] new RM replaced this: None
Sep 22 23:21:37.931 INFO [2] Starts reconcile loop
Sep 22 23:21:37.932 INFO [0] 127.0.0.1:51174 task reports connection:true
Sep 22 23:21:37.932 INFO 517f6bf3-86c2-4511-bc4e-8840c2dfa85c WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.932 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.932 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.932 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.932 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.932 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.932 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.932 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.932 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.932 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.932 INFO Max found gen is 1
Sep 22 23:21:37.932 INFO Generation requested: 3 >= found:1
Sep 22 23:21:37.932 INFO Next flush: 1
Sep 22 23:21:37.932 INFO All extents match
Sep 22 23:21:37.932 INFO No downstairs repair required
Sep 22 23:21:37.932 INFO No initial repair work was required
Sep 22 23:21:37.932 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.932 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.932 INFO 517f6bf3-86c2-4511-bc4e-8840c2dfa85c is now active with session: 9a97b4b8-4148-45e4-bbb1-302708e60500
Sep 22 23:21:37.932 INFO Opened existing region file "/tmp/downstairs-TcKuMYwD/region.json"
Sep 22 23:21:37.932 INFO Database read version 1
Sep 22 23:21:37.932 INFO 517f6bf3-86c2-4511-bc4e-8840c2dfa85c Set Active after no repair
Sep 22 23:21:37.932 INFO Database write version 1
Sep 22 23:21:37.932 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.932 INFO Set check for repair
Sep 22 23:21:37.932 INFO [1] 127.0.0.1:59981 task reports connection:true
Sep 22 23:21:37.932 INFO 517f6bf3-86c2-4511-bc4e-8840c2dfa85c Active Active Active
Sep 22 23:21:37.932 INFO Set check for repair
Sep 22 23:21:37.932 INFO [2] 127.0.0.1:40994 task reports connection:true
Sep 22 23:21:37.932 INFO 517f6bf3-86c2-4511-bc4e-8840c2dfa85c Active Active Active
Sep 22 23:21:37.932 INFO Set check for repair
Sep 22 23:21:37.932 INFO [0] received reconcile message
Sep 22 23:21:37.932 INFO [0] All repairs completed, exit
Sep 22 23:21:37.932 INFO [0] Starts cmd_loop
Sep 22 23:21:37.932 INFO [1] received reconcile message
Sep 22 23:21:37.932 INFO [1] All repairs completed, exit
Sep 22 23:21:37.932 INFO [1] Starts cmd_loop
Sep 22 23:21:37.932 INFO [2] received reconcile message
Sep 22 23:21:37.932 INFO [2] All repairs completed, exit
Sep 22 23:21:37.932 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
The guest has requested activation
Sep 22 23:21:37.933 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 active request set
Sep 22 23:21:37.933 INFO [0] received activate with gen 3
Sep 22 23:21:37.933 INFO [0] client got ds_active_rx, promote! session 2e580a9f-7e54-44b6-83db-09df314d2c01
Sep 22 23:21:37.933 INFO [1] received activate with gen 3
Sep 22 23:21:37.933 INFO [1] client got ds_active_rx, promote! session 2e580a9f-7e54-44b6-83db-09df314d2c01
Sep 22 23:21:37.933 INFO [2] received activate with gen 3
Sep 22 23:21:37.933 INFO [2] client got ds_active_rx, promote! session 2e580a9f-7e54-44b6-83db-09df314d2c01
Sep 22 23:21:37.933 INFO Signaling to UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 8442ae1d-103c-45db-9a72-ab1fbfb096dd, gen: 2 } thread that UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 2e580a9f-7e54-44b6-83db-09df314d2c01, gen: 3 } is being promoted (read-only)
Sep 22 23:21:37.933 INFO Signaling to UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 8442ae1d-103c-45db-9a72-ab1fbfb096dd, gen: 2 } thread that UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 2e580a9f-7e54-44b6-83db-09df314d2c01, gen: 3 } is being promoted (read-only)
test test::integration_test_two_layers_write_unwritten ... ok
Sep 22 23:21:37.933 INFO Signaling to UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 8442ae1d-103c-45db-9a72-ab1fbfb096dd, gen: 2 } thread that UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 2e580a9f-7e54-44b6-83db-09df314d2c01, gen: 3 } is being promoted (read-only)
test test::integration_test_upstairs_read_only_rejects_write ... ok
Sep 22 23:21:37.933 WARN Another upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 2e580a9f-7e54-44b6-83db-09df314d2c01, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 8442ae1d-103c-45db-9a72-ab1fbfb096dd, gen: 2 }, task: main
Sep 22 23:21:37.933 INFO connection (127.0.0.1:50540): all done
Sep 22 23:21:37.933 WARN Another upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 2e580a9f-7e54-44b6-83db-09df314d2c01, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 8442ae1d-103c-45db-9a72-ab1fbfb096dd, gen: 2 }, task: main
Sep 22 23:21:37.933 INFO connection (127.0.0.1:51558): all done
Sep 22 23:21:37.933 WARN Another upstairs UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 2e580a9f-7e54-44b6-83db-09df314d2c01, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 4b7c3632-4da0-4e8d-94a7-3f194685b4d5, session_id: 8442ae1d-103c-45db-9a72-ab1fbfb096dd, gen: 2 }, task: main
Sep 22 23:21:37.933 INFO connection (127.0.0.1:34528): all done
Sep 22 23:21:37.933 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.934 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.934 INFO Created new region file "/tmp/downstairs-0A3zeEIk/region.json"
Sep 22 23:21:37.934 INFO Created new region file "/tmp/downstairs-UUom91KY/region.json"
Sep 22 23:21:37.934 ERRO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) cmd_loop saw YouAreNoLongerActive 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 2e580a9f-7e54-44b6-83db-09df314d2c01 3
Sep 22 23:21:37.934 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) Active Active Active ds_transition to Disabled
Sep 22 23:21:37.934 INFO [1] Transition from Active to Disabled
Sep 22 23:21:37.934 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 set inactive, session 8442ae1d-103c-45db-9a72-ab1fbfb096dd
Sep 22 23:21:37.934 ERRO 127.0.0.1:54505: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2, looper: 1
Sep 22 23:21:37.934 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Gone missing, transition from Disabled to Disconnected
Sep 22 23:21:37.934 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 connection to 127.0.0.1:54505 closed, looper: 1
Sep 22 23:21:37.934 ERRO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) cmd_loop saw YouAreNoLongerActive 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 2e580a9f-7e54-44b6-83db-09df314d2c01 3
Sep 22 23:21:37.934 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) Active Disconnected Active ds_transition to Disabled
Sep 22 23:21:37.934 INFO [2] Transition from Active to Disabled
Sep 22 23:21:37.934 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 set inactive, session 8442ae1d-103c-45db-9a72-ab1fbfb096dd
Sep 22 23:21:37.934 INFO UUID: 62354bbd-e3bc-4b5d-ab02-957b1e479be5
Sep 22 23:21:37.934 ERRO 127.0.0.1:38410: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2, looper: 2
Sep 22 23:21:37.934 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.934 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Gone missing, transition from Disabled to Disconnected
Sep 22 23:21:37.934 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 connection to 127.0.0.1:38410 closed, looper: 2
Sep 22 23:21:37.934 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.934 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.934 ERRO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) cmd_loop saw YouAreNoLongerActive 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 2e580a9f-7e54-44b6-83db-09df314d2c01 3
Sep 22 23:21:37.934 INFO Using address: 127.0.0.1:37503, task: main
Sep 22 23:21:37.934 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (8442ae1d-103c-45db-9a72-ab1fbfb096dd) Active Disconnected Disconnected ds_transition to Disabled
Sep 22 23:21:37.934 INFO [0] Transition from Active to Disabled
Sep 22 23:21:37.934 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 set inactive, session 8442ae1d-103c-45db-9a72-ab1fbfb096dd
Sep 22 23:21:37.935 ERRO 127.0.0.1:48404: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2, looper: 0
Sep 22 23:21:37.935 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Gone missing, transition from Disabled to Disconnected
Sep 22 23:21:37.935 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 connection to 127.0.0.1:48404 closed, looper: 0
Sep 22 23:21:37.935 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.935 WARN [1] pm_task rx.recv() is None
Sep 22 23:21:37.935 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60153, task: repair
Sep 22 23:21:37.935 INFO [1] 127.0.0.1:54505 task reports connection:false
Sep 22 23:21:37.935 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60153, task: repair
Sep 22 23:21:37.935 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Disconnected Disconnected Disconnected
Sep 22 23:21:37.935 INFO [1] 127.0.0.1:54505 task reports offline
Sep 22 23:21:37.935 INFO listening, local_addr: 127.0.0.1:60153, task: repair
Sep 22 23:21:37.935 INFO [2] 127.0.0.1:38410 task reports connection:false
Sep 22 23:21:37.935 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Disconnected Disconnected Disconnected
Sep 22 23:21:37.935 INFO [2] 127.0.0.1:38410 task reports offline
Sep 22 23:21:37.935 INFO [0] 127.0.0.1:48404 task reports connection:false
Sep 22 23:21:37.935 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Disconnected Disconnected Disconnected
Sep 22 23:21:37.935 INFO [0] 127.0.0.1:48404 task reports offline
Sep 22 23:21:37.935 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60153, task: repair
Sep 22 23:21:37.935 WARN [2] pm_task rx.recv() is None
Sep 22 23:21:37.935 WARN [0] pm_task rx.recv() is None
Sep 22 23:21:37.935 INFO Using repair address: 127.0.0.1:60153, task: main
Sep 22 23:21:37.935 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.935 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.935 INFO Opened existing region file "/tmp/downstairs-XLPx1jrJ/region.json"
Sep 22 23:21:37.935 INFO Database read version 1
Sep 22 23:21:37.935 INFO Database write version 1
Sep 22 23:21:37.935 INFO [0] downstairs client at 127.0.0.1:48404 has UUID f5ea863d-974b-4712-8f07-7666c185979b
Sep 22 23:21:37.935 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f5ea863d-974b-4712-8f07-7666c185979b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.935 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.936 INFO [1] downstairs client at 127.0.0.1:54505 has UUID a1447c8e-cf80-4bea-b730-010907ef0c7f
Sep 22 23:21:37.936 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a1447c8e-cf80-4bea-b730-010907ef0c7f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.936 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.936 INFO [2] downstairs client at 127.0.0.1:38410 has UUID 1b0c673f-012e-4936-8109-087022151153
Sep 22 23:21:37.936 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1b0c673f-012e-4936-8109-087022151153, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.936 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitActive WaitActive WaitActive
Sep 22 23:21:37.936 INFO Upstairs starts
Sep 22 23:21:37.936 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:37.936 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.936 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.936 INFO Crucible stats registered with UUID: 8db920ae-ce39-4f1d-b015-8a9c440a0a0a
Sep 22 23:21:37.936 INFO Crucible 8db920ae-ce39-4f1d-b015-8a9c440a0a0a has session id: e74aff3f-2e11-4449-adea-83055a3ce4f5
Sep 22 23:21:37.936 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.936 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.936 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.936 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.936 INFO [0] connecting to 127.0.0.1:60187, looper: 0
Sep 22 23:21:37.936 INFO [1] connecting to 127.0.0.1:64895, looper: 1
Sep 22 23:21:37.937 INFO [2] connecting to 127.0.0.1:37503, looper: 2
Sep 22 23:21:37.937 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.937 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.937 INFO Flush timeout: 0.5
Sep 22 23:21:37.937 INFO accepted connection from 127.0.0.1:59290, task: main
Sep 22 23:21:37.937 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:37.937 INFO accepted connection from 127.0.0.1:50925, task: main
Sep 22 23:21:37.937 INFO accepted connection from 127.0.0.1:64950, task: main
Sep 22 23:21:37.937 INFO [1] 8db920ae-ce39-4f1d-b015-8a9c440a0a0a looper connected, looper: 1
Sep 22 23:21:37.937 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.937 INFO [1] Proc runs for 127.0.0.1:64895 in state New
Sep 22 23:21:37.937 INFO [0] 8db920ae-ce39-4f1d-b015-8a9c440a0a0a looper connected, looper: 0
Sep 22 23:21:37.937 INFO [0] Proc runs for 127.0.0.1:60187 in state New
Sep 22 23:21:37.937 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:37.937 INFO [2] 8db920ae-ce39-4f1d-b015-8a9c440a0a0a looper connected, looper: 2
Sep 22 23:21:37.937 INFO [2] Proc runs for 127.0.0.1:37503 in state New
Sep 22 23:21:37.937 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.938 INFO [0] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (2e580a9f-7e54-44b6-83db-09df314d2c01) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.938 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.938 WARN [0] new RM replaced this: None
Sep 22 23:21:37.938 INFO [0] Starts reconcile loop
Sep 22 23:21:37.938 INFO [1] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (2e580a9f-7e54-44b6-83db-09df314d2c01) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.938 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.938 WARN [1] new RM replaced this: None
Sep 22 23:21:37.938 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.938 INFO [1] Starts reconcile loop
Sep 22 23:21:37.938 INFO [2] 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 (2e580a9f-7e54-44b6-83db-09df314d2c01) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.938 INFO Created new region file "/tmp/downstairs-XDMkkT95/region.json"
Sep 22 23:21:37.938 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.938 WARN [2] new RM replaced this: None
Sep 22 23:21:37.938 INFO [2] Starts reconcile loop
Sep 22 23:21:37.938 INFO [0] 127.0.0.1:48404 task reports connection:true
Sep 22 23:21:37.938 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.938 INFO UUID: 68909a60-f6e3-4357-be0b-275dba4d89d7
Sep 22 23:21:37.938 INFO [0]R  flush_numbers: [1, 1]
Sep 22 23:21:37.938 INFO [0]R  generation: [1, 1]
Sep 22 23:21:37.938 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.938 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.938 INFO [1]R  flush_numbers: [1, 1]
Sep 22 23:21:37.938 INFO [1]R  generation: [1, 1]
Sep 22 23:21:37.938 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.938 INFO [2]R  flush_numbers: [1, 1]
Sep 22 23:21:37.938 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.938 INFO [2]R  generation: [1, 1]
Sep 22 23:21:37.938 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.938 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.938 INFO Using address: 127.0.0.1:39334, task: main
Sep 22 23:21:37.938 INFO Max found gen is 2
Sep 22 23:21:37.938 INFO Generation requested: 3 >= found:2
Sep 22 23:21:37.938 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.938 INFO Next flush: 2
Sep 22 23:21:37.938 INFO Opened existing region file "/tmp/downstairs-0A3zeEIk/region.json"
Sep 22 23:21:37.938 INFO All extents match
Sep 22 23:21:37.938 INFO Database read version 1
Sep 22 23:21:37.938 INFO Database write version 1
Sep 22 23:21:37.938 INFO No downstairs repair required
Sep 22 23:21:37.938 INFO No initial repair work was required
Sep 22 23:21:37.938 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.938 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 is now active with session: 2e580a9f-7e54-44b6-83db-09df314d2c01
Sep 22 23:21:37.938 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Set Active after no repair
Sep 22 23:21:37.938 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.938 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.938 INFO Opened existing region file "/tmp/downstairs-UUom91KY/region.json"
Sep 22 23:21:37.938 INFO Set check for repair
Sep 22 23:21:37.938 INFO Database read version 1
Sep 22 23:21:37.938 INFO Database write version 1
Sep 22 23:21:37.938 INFO [1] 127.0.0.1:54505 task reports connection:true
Sep 22 23:21:37.939 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Active Active Active
Sep 22 23:21:37.939 INFO Set check for repair
Sep 22 23:21:37.939 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.939 INFO [2] 127.0.0.1:38410 task reports connection:true
Sep 22 23:21:37.939 INFO 4b7c3632-4da0-4e8d-94a7-3f194685b4d5 Active Active Active
Sep 22 23:21:37.939 INFO Set check for repair
Sep 22 23:21:37.939 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61766, task: repair
Sep 22 23:21:37.939 INFO [0] received reconcile message
Sep 22 23:21:37.939 INFO [0] All repairs completed, exit
Sep 22 23:21:37.939 INFO [0] Starts cmd_loop
Sep 22 23:21:37.939 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61766, task: repair
Sep 22 23:21:37.939 INFO [1] received reconcile message
Sep 22 23:21:37.939 INFO listening, local_addr: 127.0.0.1:61766, task: repair
Sep 22 23:21:37.939 INFO [1] All repairs completed, exit
Sep 22 23:21:37.939 INFO [1] Starts cmd_loop
Sep 22 23:21:37.939 INFO [2] received reconcile message
Sep 22 23:21:37.939 INFO [2] All repairs completed, exit
Sep 22 23:21:37.939 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:37.939 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61766, task: repair
Sep 22 23:21:37.939 INFO Using repair address: 127.0.0.1:61766, task: main
Sep 22 23:21:37.939 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.939 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.940 INFO Upstairs starts
Sep 22 23:21:37.940 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.940 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.940 INFO Crucible stats registered with UUID: c34debde-f046-44c6-9183-91356e8062e6
Sep 22 23:21:37.940 INFO Crucible c34debde-f046-44c6-9183-91356e8062e6 has session id: 25d0d638-ed65-46be-a21d-837b71bb6b7b
Sep 22 23:21:37.940 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.940 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.940 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.940 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.940 INFO [0] connecting to 127.0.0.1:41387, looper: 0
Sep 22 23:21:37.940 INFO [1] connecting to 127.0.0.1:58081, looper: 1
Sep 22 23:21:37.940 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.940 INFO [2] connecting to 127.0.0.1:39334, looper: 2
Sep 22 23:21:37.940 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.940 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.940 INFO Flush timeout: 0.5
Sep 22 23:21:37.941 INFO accepted connection from 127.0.0.1:33706, task: main
Sep 22 23:21:37.941 INFO accepted connection from 127.0.0.1:48642, task: main
Sep 22 23:21:37.941 INFO accepted connection from 127.0.0.1:44693, task: main
Sep 22 23:21:37.941 INFO UUID: c0e11d40-fa1e-47cb-9523-4c205fda14a9
Sep 22 23:21:37.941 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.941 INFO [0] c34debde-f046-44c6-9183-91356e8062e6 looper connected, looper: 0
Sep 22 23:21:37.941 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.941 INFO [0] Proc runs for 127.0.0.1:41387 in state New
Sep 22 23:21:37.941 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.941 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.941 INFO Using address: 127.0.0.1:57089, task: main
Sep 22 23:21:37.941 INFO [1] c34debde-f046-44c6-9183-91356e8062e6 looper connected, looper: 1
Sep 22 23:21:37.941 INFO [1] Proc runs for 127.0.0.1:58081 in state New
Sep 22 23:21:37.941 INFO UUID: da052c76-0745-44df-96e3-d0a8c019994b
Sep 22 23:21:37.941 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.941 INFO [2] c34debde-f046-44c6-9183-91356e8062e6 looper connected, looper: 2
Sep 22 23:21:37.941 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.941 INFO [2] Proc runs for 127.0.0.1:39334 in state New
Sep 22 23:21:37.941 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.941 INFO Using address: 127.0.0.1:38958, task: main
Sep 22 23:21:37.942 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.942 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:63200, task: repair
Sep 22 23:21:37.942 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:63200, task: repair
Sep 22 23:21:37.942 INFO Connection request from c34debde-f046-44c6-9183-91356e8062e6 with version 4, task: proc
Sep 22 23:21:37.942 INFO listening, local_addr: 127.0.0.1:63200, task: repair
Sep 22 23:21:37.942 INFO upstairs UpstairsConnection { upstairs_id: c34debde-f046-44c6-9183-91356e8062e6, session_id: 1f51d95c-7aaf-4901-936b-d4d4f0f07c99, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.942 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.942 INFO Connection request from c34debde-f046-44c6-9183-91356e8062e6 with version 4, task: proc
Sep 22 23:21:37.942 INFO upstairs UpstairsConnection { upstairs_id: c34debde-f046-44c6-9183-91356e8062e6, session_id: 1f51d95c-7aaf-4901-936b-d4d4f0f07c99, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.942 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:63200, task: repair
Sep 22 23:21:37.942 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61420, task: repair
Sep 22 23:21:37.942 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.942 INFO Using repair address: 127.0.0.1:63200, task: main
Sep 22 23:21:37.942 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.942 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61420, task: repair
Sep 22 23:21:37.942 INFO Connection request from c34debde-f046-44c6-9183-91356e8062e6 with version 4, task: proc
Sep 22 23:21:37.942 INFO upstairs UpstairsConnection { upstairs_id: c34debde-f046-44c6-9183-91356e8062e6, session_id: 1f51d95c-7aaf-4901-936b-d4d4f0f07c99, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.942 INFO listening, local_addr: 127.0.0.1:61420, task: repair
Sep 22 23:21:37.942 DEBG [1] Read already AckReady 1000, : downstairs
The guest has requested activation
Sep 22 23:21:37.942 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61420, task: repair
Sep 22 23:21:37.942 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.942 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.942 INFO Using repair address: 127.0.0.1:61420, task: main
Sep 22 23:21:37.942 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.942 DEBG up_ds_listen was notified
Sep 22 23:21:37.942 INFO [0] c34debde-f046-44c6-9183-91356e8062e6 (1f51d95c-7aaf-4901-936b-d4d4f0f07c99) New New New ds_transition to WaitActive
Sep 22 23:21:37.942 DEBG up_ds_listen process 1000
Sep 22 23:21:37.942 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.942 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.942 INFO Created new region file "/tmp/downstairs-YjjeJN3C/region.json"
Sep 22 23:21:37.942 INFO [1] c34debde-f046-44c6-9183-91356e8062e6 (1f51d95c-7aaf-4901-936b-d4d4f0f07c99) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.942 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.942 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.942 INFO [2] c34debde-f046-44c6-9183-91356e8062e6 (1f51d95c-7aaf-4901-936b-d4d4f0f07c99) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.942 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.943 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.943 INFO Opened existing region file "/tmp/downstairs-XDMkkT95/region.json"
Sep 22 23:21:37.943 INFO Database read version 1
Sep 22 23:21:37.943 INFO Database write version 1
Sep 22 23:21:37.943 INFO c34debde-f046-44c6-9183-91356e8062e6 active request set
Sep 22 23:21:37.943 INFO Created new region file "/tmp/downstairs-m476cQbB/region.json"
Sep 22 23:21:37.943 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.943 INFO [0] received activate with gen 1
Sep 22 23:21:37.943 INFO [0] client got ds_active_rx, promote! session 1f51d95c-7aaf-4901-936b-d4d4f0f07c99
Sep 22 23:21:37.943 INFO [1] received activate with gen 1
Sep 22 23:21:37.943 INFO [1] client got ds_active_rx, promote! session 1f51d95c-7aaf-4901-936b-d4d4f0f07c99
Sep 22 23:21:37.943 DEBG IO Read  1000 has deps []
Sep 22 23:21:37.943 INFO [2] received activate with gen 1
Sep 22 23:21:37.943 INFO [2] client got ds_active_rx, promote! session 1f51d95c-7aaf-4901-936b-d4d4f0f07c99
Sep 22 23:21:37.943 INFO UpstairsConnection { upstairs_id: c34debde-f046-44c6-9183-91356e8062e6, session_id: 1f51d95c-7aaf-4901-936b-d4d4f0f07c99, gen: 1 } is now active (read-write)
Sep 22 23:21:37.943 INFO UpstairsConnection { upstairs_id: c34debde-f046-44c6-9183-91356e8062e6, session_id: 1f51d95c-7aaf-4901-936b-d4d4f0f07c99, gen: 1 } is now active (read-write)
Sep 22 23:21:37.943 INFO UpstairsConnection { upstairs_id: c34debde-f046-44c6-9183-91356e8062e6, session_id: 1f51d95c-7aaf-4901-936b-d4d4f0f07c99, gen: 1 } is now active (read-write)
Sep 22 23:21:37.943 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.944 INFO [0] downstairs client at 127.0.0.1:41387 has UUID b708efc7-8172-4645-a89a-1dd5a6e86ff4
Sep 22 23:21:37.944 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b708efc7-8172-4645-a89a-1dd5a6e86ff4, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.944 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.944 INFO c34debde-f046-44c6-9183-91356e8062e6 WaitActive WaitActive WaitActive
Sep 22 23:21:37.944 INFO [1] downstairs client at 127.0.0.1:58081 has UUID 2dedfda5-97c1-427a-8b74-5c08fb2c233b
Sep 22 23:21:37.944 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2dedfda5-97c1-427a-8b74-5c08fb2c233b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.944 INFO c34debde-f046-44c6-9183-91356e8062e6 WaitActive WaitActive WaitActive
Sep 22 23:21:37.944 INFO [2] downstairs client at 127.0.0.1:39334 has UUID 68909a60-f6e3-4357-be0b-275dba4d89d7
Sep 22 23:21:37.944 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 68909a60-f6e3-4357-be0b-275dba4d89d7, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.944 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:37.944 INFO c34debde-f046-44c6-9183-91356e8062e6 WaitActive WaitActive WaitActive
Sep 22 23:21:37.944 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.945 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.945 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.945 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.946 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.946 INFO UUID: c2905fd3-00af-4a7b-b34a-8c57c12a0633
Sep 22 23:21:37.946 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.946 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.946 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.946 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.946 INFO Using address: 127.0.0.1:42374, task: main
Sep 22 23:21:37.946 INFO [0] c34debde-f046-44c6-9183-91356e8062e6 (1f51d95c-7aaf-4901-936b-d4d4f0f07c99) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.946 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.946 WARN [0] new RM replaced this: None
Sep 22 23:21:37.946 INFO [0] Starts reconcile loop
Sep 22 23:21:37.946 INFO [1] c34debde-f046-44c6-9183-91356e8062e6 (1f51d95c-7aaf-4901-936b-d4d4f0f07c99) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.946 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.946 WARN [1] new RM replaced this: None
Sep 22 23:21:37.946 INFO [1] Starts reconcile loop
Sep 22 23:21:37.946 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.946 INFO [2] c34debde-f046-44c6-9183-91356e8062e6 (1f51d95c-7aaf-4901-936b-d4d4f0f07c99) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.947 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.947 WARN [2] new RM replaced this: None
Sep 22 23:21:37.947 INFO [2] Starts reconcile loop
Sep 22 23:21:37.947 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56257, task: repair
Sep 22 23:21:37.947 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56257, task: repair
Sep 22 23:21:37.947 INFO [0] 127.0.0.1:41387 task reports connection:true
Sep 22 23:21:37.947 INFO c34debde-f046-44c6-9183-91356e8062e6 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.947 INFO listening, local_addr: 127.0.0.1:56257, task: repair
Sep 22 23:21:37.947 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.947 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.947 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.947 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.947 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.947 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.947 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.947 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.947 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.947 INFO Max found gen is 1
Sep 22 23:21:37.947 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.947 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.947 INFO Next flush: 1
Sep 22 23:21:37.947 INFO Opened existing region file "/tmp/downstairs-YjjeJN3C/region.json"
Sep 22 23:21:37.947 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56257, task: repair
Sep 22 23:21:37.947 INFO All extents match
Sep 22 23:21:37.947 INFO Database read version 1
Sep 22 23:21:37.947 INFO No downstairs repair required
Sep 22 23:21:37.947 INFO Database write version 1
Sep 22 23:21:37.947 INFO Using repair address: 127.0.0.1:56257, task: main
Sep 22 23:21:37.947 INFO No initial repair work was required
Sep 22 23:21:37.947 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.947 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.947 INFO c34debde-f046-44c6-9183-91356e8062e6 is now active with session: 1f51d95c-7aaf-4901-936b-d4d4f0f07c99
Sep 22 23:21:37.947 INFO c34debde-f046-44c6-9183-91356e8062e6 Set Active after no repair
Sep 22 23:21:37.947 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.947 INFO Set check for repair
Sep 22 23:21:37.947 INFO [1] 127.0.0.1:58081 task reports connection:true
Sep 22 23:21:37.947 INFO c34debde-f046-44c6-9183-91356e8062e6 Active Active Active
Sep 22 23:21:37.947 INFO Set check for repair
Sep 22 23:21:37.947 INFO [2] 127.0.0.1:39334 task reports connection:true
Sep 22 23:21:37.947 INFO c34debde-f046-44c6-9183-91356e8062e6 Active Active Active
Sep 22 23:21:37.947 INFO Set check for repair
Sep 22 23:21:37.947 INFO [0] received reconcile message
Sep 22 23:21:37.947 INFO [0] All repairs completed, exit
Sep 22 23:21:37.947 INFO [0] Starts cmd_loop
Sep 22 23:21:37.947 INFO [1] received reconcile message
Sep 22 23:21:37.947 INFO [1] All repairs completed, exit
Sep 22 23:21:37.947 INFO [1] Starts cmd_loop
Sep 22 23:21:37.947 INFO Connection request from 8db920ae-ce39-4f1d-b015-8a9c440a0a0a with version 4, task: proc
Sep 22 23:21:37.947 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.947 INFO upstairs UpstairsConnection { upstairs_id: 8db920ae-ce39-4f1d-b015-8a9c440a0a0a, session_id: 10e6c626-e9aa-4c08-8604-f77440ade166, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.947 INFO Opened existing region file "/tmp/downstairs-m476cQbB/region.json"
Sep 22 23:21:37.947 INFO [2] received reconcile message
Sep 22 23:21:37.948 INFO Database read version 1
Sep 22 23:21:37.948 INFO [2] All repairs completed, exit
Sep 22 23:21:37.948 INFO Database write version 1
Sep 22 23:21:37.948 INFO [2] Starts cmd_loop
Sep 22 23:21:37.948 INFO Connection request from 8db920ae-ce39-4f1d-b015-8a9c440a0a0a with version 4, task: proc
The guest has finished waiting for activation
Sep 22 23:21:37.948 INFO upstairs UpstairsConnection { upstairs_id: 8db920ae-ce39-4f1d-b015-8a9c440a0a0a, session_id: 10e6c626-e9aa-4c08-8604-f77440ade166, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.948 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:37.948 INFO Connection request from 8db920ae-ce39-4f1d-b015-8a9c440a0a0a with version 4, task: proc
Sep 22 23:21:37.948 INFO upstairs UpstairsConnection { upstairs_id: 8db920ae-ce39-4f1d-b015-8a9c440a0a0a, session_id: 10e6c626-e9aa-4c08-8604-f77440ade166, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.948 WARN c34debde-f046-44c6-9183-91356e8062e6 request to replace downstairs 127.0.0.1:41387 with 127.0.0.1:58081
Sep 22 23:21:37.948 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.948 INFO c34debde-f046-44c6-9183-91356e8062e6 found old target: 127.0.0.1:41387 at 0
Sep 22 23:21:37.948 INFO c34debde-f046-44c6-9183-91356e8062e6 found new target: 127.0.0.1:58081 at 1
Sep 22 23:21:37.948 WARN 8db920ae-ce39-4f1d-b015-8a9c440a0a0a request to replace downstairs 127.0.0.1:60187 with 127.0.0.1:42374
Sep 22 23:21:37.948 INFO 8db920ae-ce39-4f1d-b015-8a9c440a0a0a found old target: 127.0.0.1:60187 at 0
Sep 22 23:21:37.948 INFO 8db920ae-ce39-4f1d-b015-8a9c440a0a0a replacing old: 127.0.0.1:60187 at 0
Sep 22 23:21:37.948 INFO [0] client skip 0 in process jobs because fault, : downstairs
Sep 22 23:21:37.948 INFO [0] changed 0 jobs to fault skipped, : downstairs
Sep 22 23:21:37.948 INFO [0] 8db920ae-ce39-4f1d-b015-8a9c440a0a0a (10e6c626-e9aa-4c08-8604-f77440ade166) New New New ds_transition to Replacing
Sep 22 23:21:37.948 WARN c34debde-f046-44c6-9183-91356e8062e6 request to replace downstairs 127.0.0.1:58081 with 127.0.0.1:41387
Sep 22 23:21:37.948 INFO [0] Transition from New to Replacing
Sep 22 23:21:37.948 INFO c34debde-f046-44c6-9183-91356e8062e6 found new target: 127.0.0.1:41387 at 0
Sep 22 23:21:37.948 INFO c34debde-f046-44c6-9183-91356e8062e6 found old target: 127.0.0.1:58081 at 1
Sep 22 23:21:37.950 INFO UUID: a8a89756-4dfe-4289-90cc-a00c710e1634
Sep 22 23:21:37.950 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.950 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.950 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.950 INFO Using address: 127.0.0.1:37739, task: main
Sep 22 23:21:37.950 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.951 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.951 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48197, task: repair
Sep 22 23:21:37.951 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48197, task: repair
Sep 22 23:21:37.951 INFO listening, local_addr: 127.0.0.1:48197, task: repair
Sep 22 23:21:37.951 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48197, task: repair
Sep 22 23:21:37.951 INFO Using repair address: 127.0.0.1:48197, task: main
Sep 22 23:21:37.951 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.951 INFO UUID: fc6d0cf8-ea6a-447c-b146-12fc49b9f0fc
Sep 22 23:21:37.951 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.951 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.952 INFO Created new region file "/tmp/downstairs-sOjaheCu/region.json"
Sep 22 23:21:37.952 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.952 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.952 INFO Using address: 127.0.0.1:49673, task: main
Sep 22 23:21:37.952 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.952 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:63439, task: repair
Sep 22 23:21:37.952 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:63439, task: repair
Sep 22 23:21:37.952 INFO listening, local_addr: 127.0.0.1:63439, task: repair
Sep 22 23:21:37.952 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:37.953 DEBG up_ds_listen was notified
Sep 22 23:21:37.953 DEBG up_ds_listen process 1000
Sep 22 23:21:37.953 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:63439, task: repair
Sep 22 23:21:37.953 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.953 INFO Using repair address: 127.0.0.1:63439, task: main
Sep 22 23:21:37.953 INFO No SSL acceptor configured, task: main
test test::integration_test_volume_replace_active ... ok
Sep 22 23:21:37.953 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.953 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.953 INFO Created new region file "/tmp/downstairs-bHFzvnWo/region.json"
Sep 22 23:21:37.953 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.953 INFO Created new region file "/tmp/downstairs-wZJpXBn2/region.json"
test test::integration_test_volume_inactive_replace_downstairs ... ok
Sep 22 23:21:37.954 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:21:37.954 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.954 INFO Created new region file "/tmp/downstairs-iZzJ56Ro/region.json"
Sep 22 23:21:37.955 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:3
Sep 22 23:21:37.955 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:3
Sep 22 23:21:37.955 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:3
Sep 22 23:21:37.955 DEBG up_ds_listen was notified
Sep 22 23:21:37.955 DEBG up_ds_listen process 1001
Sep 22 23:21:37.955 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.955 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:21:37.955 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.955 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.956 INFO Opened existing region file "/tmp/downstairs-sOjaheCu/region.json"
Sep 22 23:21:37.956 INFO Database read version 1
Sep 22 23:21:37.956 INFO Database write version 1
Sep 22 23:21:37.956 DEBG IO Write 1002 has deps []
Sep 22 23:21:37.956 DEBG up_ds_listen was notified
Sep 22 23:21:37.956 DEBG up_ds_listen process 1002
Sep 22 23:21:37.956 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:37.956 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.956 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.956 INFO Opened existing region file "/tmp/downstairs-bHFzvnWo/region.json"
Sep 22 23:21:37.956 INFO Database read version 1
Sep 22 23:21:37.956 INFO Database write version 1
Sep 22 23:21:37.957 INFO UUID: 6e7dc3e7-8b4b-4591-9b7c-15aab164f030
Sep 22 23:21:37.957 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.957 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.957 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.957 INFO Using address: 127.0.0.1:49483, task: main
Sep 22 23:21:37.957 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.957 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:63219, task: repair
Sep 22 23:21:37.957 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:63219, task: repair
Sep 22 23:21:37.957 INFO listening, local_addr: 127.0.0.1:63219, task: repair
Sep 22 23:21:37.957 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:37.957 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:63219, task: repair
Sep 22 23:21:37.957 INFO Using repair address: 127.0.0.1:63219, task: main
Sep 22 23:21:37.957 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.958 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:37.958 INFO Upstairs starts
Sep 22 23:21:37.958 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.958 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.958 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.958 INFO Opened existing region file "/tmp/downstairs-iZzJ56Ro/region.json"
Sep 22 23:21:37.958 INFO Crucible stats registered with UUID: eab93731-ef60-499c-b430-9f7d2df0dd78
Sep 22 23:21:37.958 INFO Database read version 1
Sep 22 23:21:37.958 INFO Crucible eab93731-ef60-499c-b430-9f7d2df0dd78 has session id: fa94d668-b6ae-4230-8587-284ca20af7fa
Sep 22 23:21:37.958 INFO Database write version 1
Sep 22 23:21:37.958 INFO UUID: a376e236-d708-4e3c-a463-d40be79bcc36
Sep 22 23:21:37.958 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.958 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.958 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.958 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.958 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.958 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.958 INFO Using address: 127.0.0.1:48309, task: main
Sep 22 23:21:37.958 INFO [0] connecting to 127.0.0.1:57089, looper: 0
Sep 22 23:21:37.958 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:37.958 INFO [1] connecting to 127.0.0.1:37739, looper: 1
Sep 22 23:21:37.958 INFO [2] connecting to 127.0.0.1:49483, looper: 2
Sep 22 23:21:37.958 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.958 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.958 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.958 INFO Flush timeout: 0.5
Sep 22 23:21:37.958 DEBG IO Read  1003 has deps [JobId(1002)]
Sep 22 23:21:37.958 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:37.959 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:37.959 INFO listening, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:37.959 INFO accepted connection from 127.0.0.1:48338, task: main
Sep 22 23:21:37.959 INFO accepted connection from 127.0.0.1:63069, task: main
Sep 22 23:21:37.959 INFO accepted connection from 127.0.0.1:39626, task: main
Sep 22 23:21:37.959 INFO [0] eab93731-ef60-499c-b430-9f7d2df0dd78 looper connected, looper: 0
Sep 22 23:21:37.959 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:37.959 INFO [0] Proc runs for 127.0.0.1:57089 in state New
Sep 22 23:21:37.959 INFO Using repair address: 127.0.0.1:53407, task: main
Sep 22 23:21:37.959 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.959 INFO [1] eab93731-ef60-499c-b430-9f7d2df0dd78 looper connected, looper: 1
Sep 22 23:21:37.959 INFO [1] Proc runs for 127.0.0.1:37739 in state New
Sep 22 23:21:37.959 INFO [2] eab93731-ef60-499c-b430-9f7d2df0dd78 looper connected, looper: 2
Sep 22 23:21:37.959 INFO [2] Proc runs for 127.0.0.1:49483 in state New
Sep 22 23:21:37.959 INFO Connection request from eab93731-ef60-499c-b430-9f7d2df0dd78 with version 4, task: proc
Sep 22 23:21:37.959 INFO upstairs UpstairsConnection { upstairs_id: eab93731-ef60-499c-b430-9f7d2df0dd78, session_id: 2a665520-5c33-4466-a6bf-8df41383b368, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.959 INFO Connection request from eab93731-ef60-499c-b430-9f7d2df0dd78 with version 4, task: proc
Sep 22 23:21:37.959 INFO upstairs UpstairsConnection { upstairs_id: eab93731-ef60-499c-b430-9f7d2df0dd78, session_id: 2a665520-5c33-4466-a6bf-8df41383b368, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.959 INFO Connection request from eab93731-ef60-499c-b430-9f7d2df0dd78 with version 4, task: proc
Sep 22 23:21:37.959 INFO upstairs UpstairsConnection { upstairs_id: eab93731-ef60-499c-b430-9f7d2df0dd78, session_id: 2a665520-5c33-4466-a6bf-8df41383b368, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.959 DEBG Read      :1003 deps:[JobId(1002)] res:true
Sep 22 23:21:37.959 INFO Upstairs starts
Sep 22 23:21:37.959 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:37.959 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:37.959 INFO Crucible stats registered with UUID: 52fe8d40-333d-4b64-8663-2adf0476947f
The guest has requested activation
Sep 22 23:21:37.959 INFO Crucible 52fe8d40-333d-4b64-8663-2adf0476947f has session id: 1000716e-0f09-43f5-b2f6-22b09aa4abd1
Sep 22 23:21:37.960 INFO [0] eab93731-ef60-499c-b430-9f7d2df0dd78 (2a665520-5c33-4466-a6bf-8df41383b368) New New New ds_transition to WaitActive
Sep 22 23:21:37.960 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.960 INFO [1] eab93731-ef60-499c-b430-9f7d2df0dd78 (2a665520-5c33-4466-a6bf-8df41383b368) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.960 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.960 DEBG Read      :1003 deps:[JobId(1002)] res:true
Sep 22 23:21:37.960 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.960 INFO [2] eab93731-ef60-499c-b430-9f7d2df0dd78 (2a665520-5c33-4466-a6bf-8df41383b368) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.960 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.960 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.960 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.960 INFO eab93731-ef60-499c-b430-9f7d2df0dd78 active request set
Sep 22 23:21:37.960 INFO [0] connecting to 127.0.0.1:38958, looper: 0
Sep 22 23:21:37.960 INFO [0] received activate with gen 1
Sep 22 23:21:37.960 INFO [0] client got ds_active_rx, promote! session 2a665520-5c33-4466-a6bf-8df41383b368
Sep 22 23:21:37.960 INFO [1] received activate with gen 1
Sep 22 23:21:37.960 INFO [1] client got ds_active_rx, promote! session 2a665520-5c33-4466-a6bf-8df41383b368
Sep 22 23:21:37.960 INFO [1] connecting to 127.0.0.1:49673, looper: 1
Sep 22 23:21:37.960 INFO [2] received activate with gen 1
Sep 22 23:21:37.960 DEBG Read      :1003 deps:[JobId(1002)] res:true
Sep 22 23:21:37.960 INFO [2] client got ds_active_rx, promote! session 2a665520-5c33-4466-a6bf-8df41383b368
Sep 22 23:21:37.960 INFO [2] connecting to 127.0.0.1:48309, looper: 2
Sep 22 23:21:37.960 INFO UpstairsConnection { upstairs_id: eab93731-ef60-499c-b430-9f7d2df0dd78, session_id: 2a665520-5c33-4466-a6bf-8df41383b368, gen: 1 } is now active (read-write)
Sep 22 23:21:37.960 INFO UpstairsConnection { upstairs_id: eab93731-ef60-499c-b430-9f7d2df0dd78, session_id: 2a665520-5c33-4466-a6bf-8df41383b368, gen: 1 } is now active (read-write)
Sep 22 23:21:37.960 INFO up_listen starts, task: up_listen
Sep 22 23:21:37.960 INFO Wait for all three downstairs to come online
Sep 22 23:21:37.960 INFO Flush timeout: 0.5
Sep 22 23:21:37.960 INFO UpstairsConnection { upstairs_id: eab93731-ef60-499c-b430-9f7d2df0dd78, session_id: 2a665520-5c33-4466-a6bf-8df41383b368, gen: 1 } is now active (read-write)
Sep 22 23:21:37.960 INFO UUID: 566f4276-dd26-4651-b2d1-ce64bf5a08b2
Sep 22 23:21:37.960 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.960 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.960 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.960 INFO accepted connection from 127.0.0.1:35663, task: main
Sep 22 23:21:37.960 INFO Using address: 127.0.0.1:44460, task: main
Sep 22 23:21:37.961 INFO accepted connection from 127.0.0.1:51009, task: main
Sep 22 23:21:37.961 INFO [0] downstairs client at 127.0.0.1:57089 has UUID c0e11d40-fa1e-47cb-9523-4c205fda14a9
Sep 22 23:21:37.961 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c0e11d40-fa1e-47cb-9523-4c205fda14a9, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.961 INFO eab93731-ef60-499c-b430-9f7d2df0dd78 WaitActive WaitActive WaitActive
Sep 22 23:21:37.961 INFO accepted connection from 127.0.0.1:33595, task: main
Sep 22 23:21:37.961 INFO [1] downstairs client at 127.0.0.1:37739 has UUID a8a89756-4dfe-4289-90cc-a00c710e1634
Sep 22 23:21:37.961 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a8a89756-4dfe-4289-90cc-a00c710e1634, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.961 INFO eab93731-ef60-499c-b430-9f7d2df0dd78 WaitActive WaitActive WaitActive
Sep 22 23:21:37.961 INFO [0] 52fe8d40-333d-4b64-8663-2adf0476947f looper connected, looper: 0
Sep 22 23:21:37.961 INFO [0] Proc runs for 127.0.0.1:38958 in state New
Sep 22 23:21:37.961 INFO [2] downstairs client at 127.0.0.1:49483 has UUID 6e7dc3e7-8b4b-4591-9b7c-15aab164f030
Sep 22 23:21:37.961 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6e7dc3e7-8b4b-4591-9b7c-15aab164f030, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.961 INFO eab93731-ef60-499c-b430-9f7d2df0dd78 WaitActive WaitActive WaitActive
Sep 22 23:21:37.961 INFO [1] 52fe8d40-333d-4b64-8663-2adf0476947f looper connected, looper: 1
Sep 22 23:21:37.961 INFO [1] Proc runs for 127.0.0.1:49673 in state New
Sep 22 23:21:37.961 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.961 INFO [2] 52fe8d40-333d-4b64-8663-2adf0476947f looper connected, looper: 2
Sep 22 23:21:37.961 INFO [2] Proc runs for 127.0.0.1:48309 in state New
Sep 22 23:21:37.961 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.961 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.961 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51286, task: repair
Sep 22 23:21:37.961 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51286, task: repair
Sep 22 23:21:37.961 INFO listening, local_addr: 127.0.0.1:51286, task: repair
Sep 22 23:21:37.961 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.961 INFO Connection request from 52fe8d40-333d-4b64-8663-2adf0476947f with version 4, task: proc
Sep 22 23:21:37.961 INFO upstairs UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.961 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.961 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51286, task: repair
Sep 22 23:21:37.961 INFO Using repair address: 127.0.0.1:51286, task: main
Sep 22 23:21:37.961 INFO Connection request from 52fe8d40-333d-4b64-8663-2adf0476947f with version 4, task: proc
Sep 22 23:21:37.962 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.962 INFO upstairs UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.962 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.962 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:21:37.962 INFO Connection request from 52fe8d40-333d-4b64-8663-2adf0476947f with version 4, task: proc
Sep 22 23:21:37.962 INFO upstairs UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:37.962 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.962 INFO current number of open files limit 65536 is already the maximum
The guest has requested activation
Sep 22 23:21:37.962 INFO Created new region file "/tmp/downstairs-NuYuZPJM/region.json"
Sep 22 23:21:37.962 INFO [0] eab93731-ef60-499c-b430-9f7d2df0dd78 (2a665520-5c33-4466-a6bf-8df41383b368) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.962 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.962 WARN [0] new RM replaced this: None
Sep 22 23:21:37.962 INFO [0] Starts reconcile loop
Sep 22 23:21:37.962 INFO [0] 52fe8d40-333d-4b64-8663-2adf0476947f (fc0a5f14-ae91-4f48-aded-c2a3fecef8a0) New New New ds_transition to WaitActive
Sep 22 23:21:37.962 INFO [0] Transition from New to WaitActive
Sep 22 23:21:37.962 INFO [1] eab93731-ef60-499c-b430-9f7d2df0dd78 (2a665520-5c33-4466-a6bf-8df41383b368) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.962 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.962 WARN [1] new RM replaced this: None
Sep 22 23:21:37.962 INFO [1] Starts reconcile loop
Sep 22 23:21:37.962 INFO [1] 52fe8d40-333d-4b64-8663-2adf0476947f (fc0a5f14-ae91-4f48-aded-c2a3fecef8a0) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:37.962 INFO [1] Transition from New to WaitActive
Sep 22 23:21:37.962 INFO [2] eab93731-ef60-499c-b430-9f7d2df0dd78 (2a665520-5c33-4466-a6bf-8df41383b368) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.962 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.962 WARN [2] new RM replaced this: None
Sep 22 23:21:37.962 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:21:37.962 INFO [2] Starts reconcile loop
Sep 22 23:21:37.962 INFO [0] 127.0.0.1:57089 task reports connection:true
Sep 22 23:21:37.962 INFO [2] 52fe8d40-333d-4b64-8663-2adf0476947f (fc0a5f14-ae91-4f48-aded-c2a3fecef8a0) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:37.962 INFO eab93731-ef60-499c-b430-9f7d2df0dd78 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.962 INFO [2] Transition from New to WaitActive
Sep 22 23:21:37.962 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.962 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.962 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.962 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.962 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.962 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.962 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.962 INFO 52fe8d40-333d-4b64-8663-2adf0476947f active request set
Sep 22 23:21:37.962 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.962 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.962 INFO Max found gen is 1
Sep 22 23:21:37.962 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.962 INFO Next flush: 1
Sep 22 23:21:37.962 INFO All extents match
Sep 22 23:21:37.962 INFO No downstairs repair required
Sep 22 23:21:37.962 INFO No initial repair work was required
Sep 22 23:21:37.962 INFO [0] received activate with gen 1
Sep 22 23:21:37.962 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.962 INFO [0] client got ds_active_rx, promote! session fc0a5f14-ae91-4f48-aded-c2a3fecef8a0
Sep 22 23:21:37.962 INFO eab93731-ef60-499c-b430-9f7d2df0dd78 is now active with session: 2a665520-5c33-4466-a6bf-8df41383b368
Sep 22 23:21:37.962 INFO eab93731-ef60-499c-b430-9f7d2df0dd78 Set Active after no repair
Sep 22 23:21:37.962 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.962 INFO Set check for repair
Sep 22 23:21:37.962 INFO [1] received activate with gen 1
Sep 22 23:21:37.962 INFO [1] 127.0.0.1:37739 task reports connection:true
Sep 22 23:21:37.962 INFO [1] client got ds_active_rx, promote! session fc0a5f14-ae91-4f48-aded-c2a3fecef8a0
Sep 22 23:21:37.962 INFO eab93731-ef60-499c-b430-9f7d2df0dd78 Active Active Active
Sep 22 23:21:37.962 INFO Set check for repair
Sep 22 23:21:37.962 INFO [2] 127.0.0.1:49483 task reports connection:true
Sep 22 23:21:37.963 INFO eab93731-ef60-499c-b430-9f7d2df0dd78 Active Active Active
Sep 22 23:21:37.963 INFO [2] received activate with gen 1
Sep 22 23:21:37.963 INFO Set check for repair
Sep 22 23:21:37.963 INFO [2] client got ds_active_rx, promote! session fc0a5f14-ae91-4f48-aded-c2a3fecef8a0
Sep 22 23:21:37.963 INFO [0] received reconcile message
Sep 22 23:21:37.963 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:21:37.963 INFO [0] All repairs completed, exit
Sep 22 23:21:37.963 DEBG up_ds_listen was notified
Sep 22 23:21:37.963 DEBG up_ds_listen process 1003
Sep 22 23:21:37.963 INFO [0] Starts cmd_loop
Sep 22 23:21:37.963 INFO UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } is now active (read-write)
Sep 22 23:21:37.963 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:37.963 INFO [1] received reconcile message
Sep 22 23:21:37.963 INFO [1] All repairs completed, exit
Sep 22 23:21:37.963 INFO [1] Starts cmd_loop
Sep 22 23:21:37.963 INFO UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } is now active (read-write)
Sep 22 23:21:37.963 INFO [2] received reconcile message
Sep 22 23:21:37.963 INFO [2] All repairs completed, exit
Sep 22 23:21:37.963 INFO [2] Starts cmd_loop
Sep 22 23:21:37.963 INFO UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } is now active (read-write)
The guest has finished waiting for activation
Sep 22 23:21:37.963 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.963 INFO Created new region file "/tmp/downstairs-DzLzCdxJ/region.json"
Sep 22 23:21:37.963 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.963 INFO [0] downstairs client at 127.0.0.1:38958 has UUID da052c76-0745-44df-96e3-d0a8c019994b
Sep 22 23:21:37.963 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: da052c76-0745-44df-96e3-d0a8c019994b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.963 DEBG IO Read  1001 has deps []
Sep 22 23:21:37.963 INFO 52fe8d40-333d-4b64-8663-2adf0476947f WaitActive WaitActive WaitActive
Sep 22 23:21:37.964 INFO [1] downstairs client at 127.0.0.1:49673 has UUID fc6d0cf8-ea6a-447c-b146-12fc49b9f0fc
Sep 22 23:21:37.964 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: fc6d0cf8-ea6a-447c-b146-12fc49b9f0fc, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.964 INFO 52fe8d40-333d-4b64-8663-2adf0476947f WaitActive WaitActive WaitActive
Sep 22 23:21:37.964 INFO [2] downstairs client at 127.0.0.1:48309 has UUID a376e236-d708-4e3c-a463-d40be79bcc36
Sep 22 23:21:37.964 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: a376e236-d708-4e3c-a463-d40be79bcc36, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:37.964 INFO 52fe8d40-333d-4b64-8663-2adf0476947f WaitActive WaitActive WaitActive
Sep 22 23:21:37.964 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.965 DEBG Read      :1001 deps:[] res:true
Sep 22 23:21:37.965 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.965 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.965 DEBG Read      :1001 deps:[] res:true
Sep 22 23:21:37.965 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.965 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:37.966 DEBG Read      :1001 deps:[] res:true
Sep 22 23:21:37.966 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:37.966 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.966 INFO Opened existing region file "/tmp/downstairs-NuYuZPJM/region.json"
Sep 22 23:21:37.966 INFO [0] 52fe8d40-333d-4b64-8663-2adf0476947f (fc0a5f14-ae91-4f48-aded-c2a3fecef8a0) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.966 INFO Database read version 1
Sep 22 23:21:37.966 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.966 INFO Database write version 1
Sep 22 23:21:37.966 WARN [0] new RM replaced this: None
Sep 22 23:21:37.966 INFO [0] Starts reconcile loop
Sep 22 23:21:37.966 INFO [1] 52fe8d40-333d-4b64-8663-2adf0476947f (fc0a5f14-ae91-4f48-aded-c2a3fecef8a0) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.966 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.966 WARN [1] new RM replaced this: None
Sep 22 23:21:37.966 INFO [1] Starts reconcile loop
Sep 22 23:21:37.967 INFO [2] 52fe8d40-333d-4b64-8663-2adf0476947f (fc0a5f14-ae91-4f48-aded-c2a3fecef8a0) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:37.967 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:37.967 WARN [2] new RM replaced this: None
Sep 22 23:21:37.967 INFO [2] Starts reconcile loop
Sep 22 23:21:37.967 INFO [0] 127.0.0.1:38958 task reports connection:true
Sep 22 23:21:37.967 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.967 INFO Opened existing region file "/tmp/downstairs-Was11KKc/region.json"
Sep 22 23:21:37.967 INFO 52fe8d40-333d-4b64-8663-2adf0476947f WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:37.967 INFO Database read version 1
Sep 22 23:21:37.967 INFO Database write version 1
Sep 22 23:21:37.967 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:37.967 INFO [0]R  generation: [0, 0]
Sep 22 23:21:37.967 INFO [0]R  dirty: [false, false]
Sep 22 23:21:37.967 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:37.967 INFO [1]R  generation: [0, 0]
Sep 22 23:21:37.967 INFO [1]R  dirty: [false, false]
Sep 22 23:21:37.967 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:37.967 INFO [2]R  generation: [0, 0]
Sep 22 23:21:37.967 INFO [2]R  dirty: [false, false]
Sep 22 23:21:37.967 INFO Max found gen is 1
Sep 22 23:21:37.967 INFO Generation requested: 1 >= found:1
Sep 22 23:21:37.967 INFO Next flush: 1
Sep 22 23:21:37.967 INFO All extents match
Sep 22 23:21:37.967 INFO No downstairs repair required
Sep 22 23:21:37.967 INFO No initial repair work was required
Sep 22 23:21:37.967 INFO Set Downstairs and Upstairs active
Sep 22 23:21:37.967 INFO 52fe8d40-333d-4b64-8663-2adf0476947f is now active with session: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0
Sep 22 23:21:37.967 INFO 52fe8d40-333d-4b64-8663-2adf0476947f Set Active after no repair
Sep 22 23:21:37.967 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:37.967 INFO Set check for repair
Sep 22 23:21:37.967 INFO [1] 127.0.0.1:49673 task reports connection:true
Sep 22 23:21:37.967 INFO 52fe8d40-333d-4b64-8663-2adf0476947f Active Active Active
Sep 22 23:21:37.967 INFO Set check for repair
Sep 22 23:21:37.967 INFO [2] 127.0.0.1:48309 task reports connection:true
Sep 22 23:21:37.967 INFO 52fe8d40-333d-4b64-8663-2adf0476947f Active Active Active
Sep 22 23:21:37.967 INFO Set check for repair
Sep 22 23:21:37.967 INFO [0] received reconcile message
Sep 22 23:21:37.967 INFO [0] All repairs completed, exit
Sep 22 23:21:37.967 INFO [0] Starts cmd_loop
Sep 22 23:21:37.967 INFO [1] received reconcile message
Sep 22 23:21:37.968 INFO [1] All repairs completed, exit
Sep 22 23:21:37.968 INFO [1] Starts cmd_loop
Sep 22 23:21:37.968 INFO [2] received reconcile message
Sep 22 23:21:37.968 INFO [2] All repairs completed, exit
Sep 22 23:21:37.968 INFO [2] Starts cmd_loop
Sep 22 23:21:37.968 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.968 INFO Opened existing region file "/tmp/downstairs-DzLzCdxJ/region.json"
The guest has finished waiting for activation
Sep 22 23:21:37.968 INFO Database read version 1
Sep 22 23:21:37.968 INFO Database write version 1
Sep 22 23:21:37.970 INFO UUID: 479a8ee4-cd0f-4e61-bc88-2060ae3b8bdc
Sep 22 23:21:37.970 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.970 INFO UUID: 7a4b0c61-160e-4afd-815e-9c0c3ef81d3d
Sep 22 23:21:37.970 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.970 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.970 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.970 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.970 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.970 INFO Using address: 127.0.0.1:61097, task: main
Sep 22 23:21:37.970 INFO Using address: 127.0.0.1:32949, task: main
Sep 22 23:21:37.970 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.970 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:64906, task: repair
Sep 22 23:21:37.970 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.970 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:64906, task: repair
Sep 22 23:21:37.970 INFO listening, local_addr: 127.0.0.1:64906, task: repair
Sep 22 23:21:37.970 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44906, task: repair
Sep 22 23:21:37.970 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44906, task: repair
Sep 22 23:21:37.970 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:64906, task: repair
Sep 22 23:21:37.970 INFO listening, local_addr: 127.0.0.1:44906, task: repair
Sep 22 23:21:37.970 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:37.971 INFO Using repair address: 127.0.0.1:64906, task: main
Sep 22 23:21:37.971 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.971 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44906, task: repair
Sep 22 23:21:37.971 INFO Using repair address: 127.0.0.1:44906, task: main
Sep 22 23:21:37.971 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.971 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.971 INFO Created new region file "/tmp/downstairs-JXFR0BRF/region.json"
Sep 22 23:21:37.971 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.971 DEBG IO Write 1000 has deps []
Sep 22 23:21:37.971 INFO Created new region file "/tmp/downstairs-MD5F5T4f/region.json"
Sep 22 23:21:37.971 DEBG up_ds_listen was notified
Sep 22 23:21:37.971 DEBG up_ds_listen process 1000
Sep 22 23:21:37.971 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:37.971 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.972 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.972 INFO Created new region file "/tmp/downstairs-kiLstbeI/region.json"
Sep 22 23:21:37.973 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:37.974 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.974 INFO Opened existing region file "/tmp/downstairs-JXFR0BRF/region.json"
Sep 22 23:21:37.974 INFO Database read version 1
Sep 22 23:21:37.974 INFO Database write version 1
Sep 22 23:21:37.976 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:37.976 DEBG up_ds_listen was notified
Sep 22 23:21:37.976 DEBG up_ds_listen process 1001
Sep 22 23:21:37.976 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:37.976 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.976 INFO Opened existing region file "/tmp/downstairs-MD5F5T4f/region.json"
Sep 22 23:21:37.976 INFO Database read version 1
Sep 22 23:21:37.976 INFO Database write version 1
Sep 22 23:21:37.977 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.977 DEBG IO Flush 1004 has deps [JobId(1003), JobId(1002)]
Sep 22 23:21:37.978 INFO UUID: e0100601-6e4a-4881-b022-fa3b270481de
Sep 22 23:21:37.978 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.978 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.978 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.978 INFO Using address: 127.0.0.1:47263, task: main
Sep 22 23:21:37.978 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.978 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43250, task: repair
Sep 22 23:21:37.978 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43250, task: repair
Sep 22 23:21:37.978 INFO listening, local_addr: 127.0.0.1:43250, task: repair
Sep 22 23:21:37.978 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.978 INFO Opened existing region file "/tmp/downstairs-kiLstbeI/region.json"
Sep 22 23:21:37.978 INFO Database read version 1
Sep 22 23:21:37.978 INFO Database write version 1
Sep 22 23:21:37.978 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43250, task: repair
Sep 22 23:21:37.978 INFO Using repair address: 127.0.0.1:43250, task: main
Sep 22 23:21:37.978 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.979 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002)] res:true f:2 g:3
Sep 22 23:21:37.979 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002)] res:true f:2 g:3
Sep 22 23:21:37.979 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.979 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.979 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002)] res:true f:2 g:3
Sep 22 23:21:37.979 WARN eab93731-ef60-499c-b430-9f7d2df0dd78 request to replace downstairs 127.0.0.1:61097 with 127.0.0.1:47263
Sep 22 23:21:37.979 WARN eab93731-ef60-499c-b430-9f7d2df0dd78 downstairs 127.0.0.1:61097 not found
Sep 22 23:21:37.979 INFO UUID: 7d8c61fb-0fd3-430c-b1f3-5e7e68ad6607
Sep 22 23:21:37.979 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.979 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.979 DEBG up_ds_listen was notified
Sep 22 23:21:37.979 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.979 INFO Using address: 127.0.0.1:53630, task: main
Sep 22 23:21:37.979 DEBG up_ds_listen process 1004
Sep 22 23:21:37.979 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:21:37.979 DEBG [rc] retire 1004 clears [JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:21:37.979 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:37.980 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.980 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48976, task: repair
Sep 22 23:21:37.980 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48976, task: repair
Sep 22 23:21:37.980 INFO listening, local_addr: 127.0.0.1:48976, task: repair
Sep 22 23:21:37.980 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48976, task: repair
Sep 22 23:21:37.980 INFO Using repair address: 127.0.0.1:48976, task: main
Sep 22 23:21:37.980 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.980 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.981 INFO Created new region file "/tmp/downstairs-42faY5Vy/region.json"
Sep 22 23:21:37.981 INFO UUID: f9881bf1-86b5-438d-b94a-613d437493c1
Sep 22 23:21:37.981 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.981 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.981 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.981 INFO Using address: 127.0.0.1:62901, task: main
Sep 22 23:21:37.982 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.982 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:45053, task: repair
Sep 22 23:21:37.982 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:45053, task: repair
Sep 22 23:21:37.982 INFO listening, local_addr: 127.0.0.1:45053, task: repair
Sep 22 23:21:37.982 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:45053, task: repair
Sep 22 23:21:37.982 INFO Using repair address: 127.0.0.1:45053, task: main
Sep 22 23:21:37.982 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.984 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:37.984 WARN 52fe8d40-333d-4b64-8663-2adf0476947f request to replace downstairs 127.0.0.1:38958 with 127.0.0.1:62901
Sep 22 23:21:37.984 INFO 52fe8d40-333d-4b64-8663-2adf0476947f found old target: 127.0.0.1:38958 at 0
Sep 22 23:21:37.984 INFO 52fe8d40-333d-4b64-8663-2adf0476947f replacing old: 127.0.0.1:38958 at 0
Sep 22 23:21:37.984 INFO [0] client skip 1 in process jobs because fault, : downstairs
Sep 22 23:21:37.984 INFO [0] changed 1 jobs to fault skipped, : downstairs
Sep 22 23:21:37.984 INFO [0] 52fe8d40-333d-4b64-8663-2adf0476947f (fc0a5f14-ae91-4f48-aded-c2a3fecef8a0) Active Active Active ds_transition to Replacing
Sep 22 23:21:37.984 INFO [0] Transition from Active to Replacing
test test::integration_test_volume_replace_bad_downstairs ... ok
Sep 22 23:21:37.986 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.986 INFO Created new region file "/tmp/downstairs-GnOuEPSa/region.json"
Sep 22 23:21:37.986 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:37.986 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.987 INFO Opened existing region file "/tmp/downstairs-42faY5Vy/region.json"
Sep 22 23:21:37.987 INFO Database read version 1
Sep 22 23:21:37.987 INFO Database write version 1
Sep 22 23:21:37.988 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:37.988 INFO UUID: 3e5893e8-0146-44bc-9dcd-87f1c9321550
Sep 22 23:21:37.988 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.989 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.989 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.989 INFO Using address: 127.0.0.1:62409, task: main
Sep 22 23:21:37.989 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.989 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49988, task: repair
Sep 22 23:21:37.989 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49988, task: repair
Sep 22 23:21:37.989 INFO listening, local_addr: 127.0.0.1:49988, task: repair
Sep 22 23:21:37.989 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:37.989 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49988, task: repair
Sep 22 23:21:37.989 INFO Using repair address: 127.0.0.1:49988, task: main
Sep 22 23:21:37.989 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.989 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.989 INFO Opened existing region file "/tmp/downstairs-GnOuEPSa/region.json"
Sep 22 23:21:37.989 INFO Database read version 1
Sep 22 23:21:37.989 INFO Database write version 1
Sep 22 23:21:37.989 WARN [0] 52fe8d40-333d-4b64-8663-2adf0476947f WARNING finish job 1000 when downstairs state:Replacing
Sep 22 23:21:37.989 WARN [0] Dropping already skipped job 1000, : downstairs
Sep 22 23:21:37.990 WARN [0] will exit pm_task, this downstairs Replacing
Sep 22 23:21:37.990 DEBG up_ds_listen was notified
Sep 22 23:21:37.990 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:21:37.990 ERRO 127.0.0.1:38958: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Replacing)), so we end too, looper: 0
Sep 22 23:21:37.990 INFO [0] 52fe8d40-333d-4b64-8663-2adf0476947f Gone missing, transition from Replacing to Replaced
Sep 22 23:21:37.990 INFO [0] 52fe8d40-333d-4b64-8663-2adf0476947f connection to 127.0.0.1:38958 closed, looper: 0
Sep 22 23:21:37.990 INFO [0] 127.0.0.1:38958 task reports connection:false
Sep 22 23:21:37.990 INFO 52fe8d40-333d-4b64-8663-2adf0476947f Replaced Active Active
Sep 22 23:21:37.990 INFO [0] 127.0.0.1:38958 task reports offline
Sep 22 23:21:37.990 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.990 WARN upstairs UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:21:37.990 WARN upstairs UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } was previously active, clearing, task: main
Sep 22 23:21:37.990 INFO connection (127.0.0.1:51009): all done
Sep 22 23:21:37.990 INFO Created new region file "/tmp/downstairs-DqCVu0Sw/region.json"
test test::integration_test_snapshot_backed_vol ... ok
Sep 22 23:21:37.991 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.991 INFO Created new region file "/tmp/downstairs-Km6l7HnV/region.json"
Sep 22 23:21:37.991 INFO UUID: 5e565886-9774-4b9d-9633-93979cb0f8ae
Sep 22 23:21:37.991 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.991 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.991 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.991 INFO Using address: 127.0.0.1:49721, task: main
Sep 22 23:21:37.991 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.991 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56432, task: repair
Sep 22 23:21:37.992 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56432, task: repair
Sep 22 23:21:37.992 INFO listening, local_addr: 127.0.0.1:56432, task: repair
Sep 22 23:21:37.992 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56432, task: repair
Sep 22 23:21:37.992 INFO Using repair address: 127.0.0.1:56432, task: main
Sep 22 23:21:37.992 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.992 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.993 INFO Created new region file "/tmp/downstairs-TlJziCp7/region.json"
Sep 22 23:21:37.994 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.994 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.994 INFO Opened existing region file "/tmp/downstairs-DqCVu0Sw/region.json"
Sep 22 23:21:37.994 INFO Opened existing region file "/tmp/downstairs-Km6l7HnV/region.json"
Sep 22 23:21:37.994 INFO Database read version 1
Sep 22 23:21:37.994 INFO Database read version 1
Sep 22 23:21:37.994 INFO Database write version 1
Sep 22 23:21:37.994 INFO Database write version 1
Sep 22 23:21:37.996 INFO UUID: fe4a84c6-9d76-42b5-9360-3287766e9e73
Sep 22 23:21:37.996 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.996 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.996 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.996 INFO Using address: 127.0.0.1:64852, task: main
Sep 22 23:21:37.996 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.996 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:59280, task: repair
Sep 22 23:21:37.996 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:59280, task: repair
Sep 22 23:21:37.996 INFO listening, local_addr: 127.0.0.1:59280, task: repair
Sep 22 23:21:37.996 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:59280, task: repair
Sep 22 23:21:37.996 INFO Using repair address: 127.0.0.1:59280, task: main
Sep 22 23:21:37.996 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.997 INFO UUID: 1d55152f-d449-4fe0-aafd-0fc27bdcca9b
Sep 22 23:21:37.997 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.997 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.997 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.997 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.997 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.997 INFO Opened existing region file "/tmp/downstairs-TlJziCp7/region.json"
Sep 22 23:21:37.997 INFO Database read version 1
Sep 22 23:21:37.997 INFO Created new region file "/tmp/downstairs-W3ZwVj5O/region.json"
Sep 22 23:21:37.997 INFO Using address: 127.0.0.1:58576, task: main
Sep 22 23:21:37.997 INFO Database write version 1
Sep 22 23:21:37.997 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.997 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61143, task: repair
Sep 22 23:21:37.997 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61143, task: repair
Sep 22 23:21:37.997 INFO listening, local_addr: 127.0.0.1:61143, task: repair
Sep 22 23:21:37.997 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61143, task: repair
Sep 22 23:21:37.997 INFO Using repair address: 127.0.0.1:61143, task: main
Sep 22 23:21:37.998 INFO No SSL acceptor configured, task: main
Sep 22 23:21:37.998 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:37.998 INFO Created new region file "/tmp/downstairs-lvGHo6t9/region.json"
Sep 22 23:21:37.999 INFO UUID: 6c7e6774-c872-46de-b630-fdc717de2268
Sep 22 23:21:37.999 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:37.999 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:37.999 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:37.999 INFO Using address: 127.0.0.1:49229, task: main
Sep 22 23:21:37.999 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:37.999 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57612, task: repair
Sep 22 23:21:37.999 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57612, task: repair
Sep 22 23:21:37.999 INFO listening, local_addr: 127.0.0.1:57612, task: repair
Sep 22 23:21:38.000 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57612, task: repair
Sep 22 23:21:38.000 INFO Using repair address: 127.0.0.1:57612, task: main
Sep 22 23:21:38.000 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.000 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.000 INFO Created new region file "/tmp/downstairs-LhVTHdCU/region.json"
Sep 22 23:21:38.001 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.001 INFO Opened existing region file "/tmp/downstairs-W3ZwVj5O/region.json"
Sep 22 23:21:38.001 INFO Database read version 1
Sep 22 23:21:38.001 INFO Database write version 1
Sep 22 23:21:38.004 INFO UUID: 2be75e90-0f81-4f50-bf88-09ce34abc850
Sep 22 23:21:38.004 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.004 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.004 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.004 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.004 INFO Opened existing region file "/tmp/downstairs-lvGHo6t9/region.json"
Sep 22 23:21:38.004 INFO Using address: 127.0.0.1:55577, task: main
Sep 22 23:21:38.004 INFO Database read version 1
Sep 22 23:21:38.004 INFO Database write version 1
Sep 22 23:21:38.004 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.004 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50556, task: repair
Sep 22 23:21:38.004 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50556, task: repair
Sep 22 23:21:38.004 INFO listening, local_addr: 127.0.0.1:50556, task: repair
Sep 22 23:21:38.004 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50556, task: repair
Sep 22 23:21:38.005 INFO Using repair address: 127.0.0.1:50556, task: main
Sep 22 23:21:38.005 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.005 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.005 INFO Opened existing region file "/tmp/downstairs-LhVTHdCU/region.json"
Sep 22 23:21:38.005 INFO Database read version 1
Sep 22 23:21:38.005 INFO Database write version 1
Sep 22 23:21:38.005 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.005 INFO Created new region file "/tmp/downstairs-3VJmaWqa/region.json"
Sep 22 23:21:38.008 INFO UUID: 709bf894-ad93-42e1-b7a3-127d2f1f8b04
Sep 22 23:21:38.008 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.008 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.008 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.008 INFO Using address: 127.0.0.1:59698, task: main
Sep 22 23:21:38.008 INFO UUID: 1839dd14-6901-4a5d-8bee-dbdeaa422af1
Sep 22 23:21:38.008 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.009 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.009 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.009 INFO Using address: 127.0.0.1:55350, task: main
Sep 22 23:21:38.009 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.009 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:36399, task: repair
Sep 22 23:21:38.009 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.009 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:36399, task: repair
Sep 22 23:21:38.009 INFO listening, local_addr: 127.0.0.1:36399, task: repair
Sep 22 23:21:38.009 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60827, task: repair
Sep 22 23:21:38.009 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60827, task: repair
Sep 22 23:21:38.009 INFO listening, local_addr: 127.0.0.1:60827, task: repair
Sep 22 23:21:38.009 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:36399, task: repair
Sep 22 23:21:38.009 INFO Using repair address: 127.0.0.1:36399, task: main
Sep 22 23:21:38.009 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.009 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60827, task: repair
Sep 22 23:21:38.009 INFO Using repair address: 127.0.0.1:60827, task: main
Sep 22 23:21:38.009 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.010 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.010 INFO Created new region file "/tmp/downstairs-cCknzOql/region.json"
Sep 22 23:21:38.010 INFO Upstairs starts
Sep 22 23:21:38.010 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.010 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.010 INFO Crucible stats registered with UUID: 459d711a-572f-4bec-bee0-901ef0300807
Sep 22 23:21:38.010 INFO Crucible 459d711a-572f-4bec-bee0-901ef0300807 has session id: 700a1ac5-add3-431a-ba59-75efa3b1f621
Sep 22 23:21:38.010 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.010 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.010 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.010 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.011 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.011 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.011 INFO [0] connecting to 127.0.0.1:44460, looper: 0
Sep 22 23:21:38.011 INFO [1] connecting to 127.0.0.1:32949, looper: 1
Sep 22 23:21:38.011 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.011 INFO Opened existing region file "/tmp/downstairs-3VJmaWqa/region.json"
Sep 22 23:21:38.011 INFO Database read version 1
Sep 22 23:21:38.011 INFO Database write version 1
Sep 22 23:21:38.011 INFO [2] connecting to 127.0.0.1:53630, looper: 2
Sep 22 23:21:38.011 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.011 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.011 INFO Flush timeout: 0.5
Sep 22 23:21:38.011 INFO accepted connection from 127.0.0.1:48984, task: main
Sep 22 23:21:38.011 INFO accepted connection from 127.0.0.1:60768, task: main
Sep 22 23:21:38.012 INFO accepted connection from 127.0.0.1:50875, task: main
Sep 22 23:21:38.012 INFO [0] 459d711a-572f-4bec-bee0-901ef0300807 looper connected, looper: 0
Sep 22 23:21:38.012 INFO [0] Proc runs for 127.0.0.1:44460 in state New
Sep 22 23:21:38.012 INFO [1] 459d711a-572f-4bec-bee0-901ef0300807 looper connected, looper: 1
Sep 22 23:21:38.012 INFO [1] Proc runs for 127.0.0.1:32949 in state New
Sep 22 23:21:38.012 INFO [2] 459d711a-572f-4bec-bee0-901ef0300807 looper connected, looper: 2
Sep 22 23:21:38.012 INFO [2] Proc runs for 127.0.0.1:53630 in state New
Sep 22 23:21:38.012 INFO Upstairs starts
Sep 22 23:21:38.012 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.012 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.012 INFO Crucible stats registered with UUID: 57a23afd-0f7e-4863-905a-9c435a295364
Sep 22 23:21:38.012 INFO Crucible 57a23afd-0f7e-4863-905a-9c435a295364 has session id: 100195fc-8b85-43fc-8631-36ee6b1ec0c4
Sep 22 23:21:38.013 INFO Connection request from 459d711a-572f-4bec-bee0-901ef0300807 with version 4, task: proc
Sep 22 23:21:38.013 INFO upstairs UpstairsConnection { upstairs_id: 459d711a-572f-4bec-bee0-901ef0300807, session_id: d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.013 INFO Connection request from 459d711a-572f-4bec-bee0-901ef0300807 with version 4, task: proc
Sep 22 23:21:38.013 INFO upstairs UpstairsConnection { upstairs_id: 459d711a-572f-4bec-bee0-901ef0300807, session_id: d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.013 INFO Connection request from 459d711a-572f-4bec-bee0-901ef0300807 with version 4, task: proc
Sep 22 23:21:38.013 INFO upstairs UpstairsConnection { upstairs_id: 459d711a-572f-4bec-bee0-901ef0300807, session_id: d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.013 INFO [0] connecting to 127.0.0.1:62409, looper: 0
Sep 22 23:21:38.013 INFO [1] connecting to 127.0.0.1:58576, looper: 1
Sep 22 23:21:38.013 INFO [2] connecting to 127.0.0.1:55350, looper: 2
Sep 22 23:21:38.013 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.013 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.013 INFO Flush timeout: 0.5
Sep 22 23:21:38.013 INFO accepted connection from 127.0.0.1:39090, task: main
Sep 22 23:21:38.013 INFO accepted connection from 127.0.0.1:56927, task: main
Sep 22 23:21:38.014 INFO accepted connection from 127.0.0.1:44673, task: main
Sep 22 23:21:38.014 INFO [0] 459d711a-572f-4bec-bee0-901ef0300807 (d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d) New New New ds_transition to WaitActive
Sep 22 23:21:38.014 INFO UUID: 5366c239-e0fc-41ac-ab9f-b458e8730135
Sep 22 23:21:38.014 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.014 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.014 INFO [1] 459d711a-572f-4bec-bee0-901ef0300807 (d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.014 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.014 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.014 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.014 INFO Using address: 127.0.0.1:49689, task: main
Sep 22 23:21:38.014 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.014 INFO [2] 459d711a-572f-4bec-bee0-901ef0300807 (d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.014 INFO Opened existing region file "/tmp/downstairs-cCknzOql/region.json"
Sep 22 23:21:38.014 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.014 INFO Database read version 1
Sep 22 23:21:38.014 INFO Database write version 1
Sep 22 23:21:38.014 INFO [0] 57a23afd-0f7e-4863-905a-9c435a295364 looper connected, looper: 0
Sep 22 23:21:38.014 INFO [0] Proc runs for 127.0.0.1:62409 in state New
Sep 22 23:21:38.014 INFO [1] 57a23afd-0f7e-4863-905a-9c435a295364 looper connected, looper: 1
Sep 22 23:21:38.014 INFO [1] Proc runs for 127.0.0.1:58576 in state New
Sep 22 23:21:38.014 INFO [2] 57a23afd-0f7e-4863-905a-9c435a295364 looper connected, looper: 2
Sep 22 23:21:38.014 INFO [2] Proc runs for 127.0.0.1:55350 in state New
Sep 22 23:21:38.014 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.014 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37669, task: repair
Sep 22 23:21:38.014 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37669, task: repair
Sep 22 23:21:38.014 INFO listening, local_addr: 127.0.0.1:37669, task: repair
Sep 22 23:21:38.014 INFO Connection request from 57a23afd-0f7e-4863-905a-9c435a295364 with version 4, task: proc
Sep 22 23:21:38.014 INFO upstairs UpstairsConnection { upstairs_id: 57a23afd-0f7e-4863-905a-9c435a295364, session_id: 465019ca-32d3-499c-bb90-0f856bcdf646, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.014 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37669, task: repair
Sep 22 23:21:38.014 INFO Using repair address: 127.0.0.1:37669, task: main
Sep 22 23:21:38.014 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.014 INFO Connection request from 57a23afd-0f7e-4863-905a-9c435a295364 with version 4, task: proc
Sep 22 23:21:38.014 INFO upstairs UpstairsConnection { upstairs_id: 57a23afd-0f7e-4863-905a-9c435a295364, session_id: 465019ca-32d3-499c-bb90-0f856bcdf646, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.015 INFO Connection request from 57a23afd-0f7e-4863-905a-9c435a295364 with version 4, task: proc
Sep 22 23:21:38.015 INFO upstairs UpstairsConnection { upstairs_id: 57a23afd-0f7e-4863-905a-9c435a295364, session_id: 465019ca-32d3-499c-bb90-0f856bcdf646, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.015 INFO [0] 57a23afd-0f7e-4863-905a-9c435a295364 (465019ca-32d3-499c-bb90-0f856bcdf646) New New New ds_transition to WaitActive
Sep 22 23:21:38.015 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.015 INFO [1] 57a23afd-0f7e-4863-905a-9c435a295364 (465019ca-32d3-499c-bb90-0f856bcdf646) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.015 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.015 INFO [2] 57a23afd-0f7e-4863-905a-9c435a295364 (465019ca-32d3-499c-bb90-0f856bcdf646) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.015 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:21:38.015 INFO 459d711a-572f-4bec-bee0-901ef0300807 active request set
Sep 22 23:21:38.015 INFO [0] received activate with gen 1
Sep 22 23:21:38.015 INFO [0] client got ds_active_rx, promote! session d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d
Sep 22 23:21:38.015 INFO Upstairs starts
Sep 22 23:21:38.015 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.015 INFO [1] received activate with gen 1
Sep 22 23:21:38.015 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.015 INFO [1] client got ds_active_rx, promote! session d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d
Sep 22 23:21:38.015 INFO Crucible stats registered with UUID: 7aa67697-0340-40b6-98b6-fb2ed6103ed6
Sep 22 23:21:38.015 INFO Crucible 7aa67697-0340-40b6-98b6-fb2ed6103ed6 has session id: ca9217d4-5a4f-4f73-90fb-a75120e3531d
Sep 22 23:21:38.015 INFO [2] received activate with gen 1
Sep 22 23:21:38.015 INFO [2] client got ds_active_rx, promote! session d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d
Sep 22 23:21:38.015 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.015 INFO UpstairsConnection { upstairs_id: 459d711a-572f-4bec-bee0-901ef0300807, session_id: d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d, gen: 1 } is now active (read-write)
Sep 22 23:21:38.015 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.015 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.016 INFO UpstairsConnection { upstairs_id: 459d711a-572f-4bec-bee0-901ef0300807, session_id: d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d, gen: 1 } is now active (read-write)
Sep 22 23:21:38.016 INFO [0] connecting to 127.0.0.1:64852, looper: 0
Sep 22 23:21:38.016 INFO UpstairsConnection { upstairs_id: 459d711a-572f-4bec-bee0-901ef0300807, session_id: d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d, gen: 1 } is now active (read-write)
Sep 22 23:21:38.016 INFO [1] connecting to 127.0.0.1:55577, looper: 1
Sep 22 23:21:38.016 INFO [2] connecting to 127.0.0.1:49689, looper: 2
Sep 22 23:21:38.016 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.016 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.016 INFO Flush timeout: 0.5
Sep 22 23:21:38.016 INFO [0] downstairs client at 127.0.0.1:44460 has UUID 566f4276-dd26-4651-b2d1-ce64bf5a08b2
Sep 22 23:21:38.016 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 566f4276-dd26-4651-b2d1-ce64bf5a08b2, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.016 INFO 459d711a-572f-4bec-bee0-901ef0300807 WaitActive WaitActive WaitActive
Sep 22 23:21:38.016 INFO [1] downstairs client at 127.0.0.1:32949 has UUID 479a8ee4-cd0f-4e61-bc88-2060ae3b8bdc
Sep 22 23:21:38.016 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 479a8ee4-cd0f-4e61-bc88-2060ae3b8bdc, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.016 INFO accepted connection from 127.0.0.1:53788, task: main
Sep 22 23:21:38.016 INFO 459d711a-572f-4bec-bee0-901ef0300807 WaitActive WaitActive WaitActive
Sep 22 23:21:38.016 INFO accepted connection from 127.0.0.1:52799, task: main
Sep 22 23:21:38.016 INFO [2] downstairs client at 127.0.0.1:53630 has UUID 7d8c61fb-0fd3-430c-b1f3-5e7e68ad6607
Sep 22 23:21:38.016 INFO accepted connection from 127.0.0.1:38518, task: main
Sep 22 23:21:38.016 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7d8c61fb-0fd3-430c-b1f3-5e7e68ad6607, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.016 INFO 459d711a-572f-4bec-bee0-901ef0300807 WaitActive WaitActive WaitActive
Sep 22 23:21:38.016 INFO [0] 7aa67697-0340-40b6-98b6-fb2ed6103ed6 looper connected, looper: 0
Sep 22 23:21:38.016 INFO [0] Proc runs for 127.0.0.1:64852 in state New
Sep 22 23:21:38.016 INFO [1] 7aa67697-0340-40b6-98b6-fb2ed6103ed6 looper connected, looper: 1
Sep 22 23:21:38.016 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.016 INFO [1] Proc runs for 127.0.0.1:55577 in state New
Sep 22 23:21:38.016 INFO [2] 7aa67697-0340-40b6-98b6-fb2ed6103ed6 looper connected, looper: 2
Sep 22 23:21:38.016 INFO [2] Proc runs for 127.0.0.1:49689 in state New
Sep 22 23:21:38.017 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.017 INFO UUID: 137bc727-67de-4ee4-a302-cef52368a28d
Sep 22 23:21:38.017 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.017 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.017 INFO Connection request from 7aa67697-0340-40b6-98b6-fb2ed6103ed6 with version 4, task: proc
Sep 22 23:21:38.017 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.017 INFO upstairs UpstairsConnection { upstairs_id: 7aa67697-0340-40b6-98b6-fb2ed6103ed6, session_id: 9ec08283-1762-4efd-adf7-8c5fe20cabd0, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.017 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.017 INFO Using address: 127.0.0.1:60338, task: main
Sep 22 23:21:38.017 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.017 INFO Connection request from 7aa67697-0340-40b6-98b6-fb2ed6103ed6 with version 4, task: proc
Sep 22 23:21:38.017 INFO upstairs UpstairsConnection { upstairs_id: 7aa67697-0340-40b6-98b6-fb2ed6103ed6, session_id: 9ec08283-1762-4efd-adf7-8c5fe20cabd0, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.017 INFO Connection request from 7aa67697-0340-40b6-98b6-fb2ed6103ed6 with version 4, task: proc
Sep 22 23:21:38.017 INFO upstairs UpstairsConnection { upstairs_id: 7aa67697-0340-40b6-98b6-fb2ed6103ed6, session_id: 9ec08283-1762-4efd-adf7-8c5fe20cabd0, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.017 INFO Current flush_numbers [0..12]: [0, 0]
The guest has requested activation
Sep 22 23:21:38.017 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.017 INFO [0] 7aa67697-0340-40b6-98b6-fb2ed6103ed6 (9ec08283-1762-4efd-adf7-8c5fe20cabd0) New New New ds_transition to WaitActive
Sep 22 23:21:38.017 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.017 INFO [1] 7aa67697-0340-40b6-98b6-fb2ed6103ed6 (9ec08283-1762-4efd-adf7-8c5fe20cabd0) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.017 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.017 INFO [0] 459d711a-572f-4bec-bee0-901ef0300807 (d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.017 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.017 WARN [0] new RM replaced this: None
Sep 22 23:21:38.017 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.017 INFO [2] 7aa67697-0340-40b6-98b6-fb2ed6103ed6 (9ec08283-1762-4efd-adf7-8c5fe20cabd0) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.017 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.017 INFO [0] Starts reconcile loop
Sep 22 23:21:38.017 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 active request set
Sep 22 23:21:38.017 INFO [1] 459d711a-572f-4bec-bee0-901ef0300807 (d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.017 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.017 WARN [1] new RM replaced this: None
Sep 22 23:21:38.017 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52052, task: repair
Sep 22 23:21:38.017 INFO [0] received activate with gen 1
Sep 22 23:21:38.017 INFO [1] Starts reconcile loop
Sep 22 23:21:38.017 INFO [0] client got ds_active_rx, promote! session 9ec08283-1762-4efd-adf7-8c5fe20cabd0
Sep 22 23:21:38.017 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52052, task: repair
Sep 22 23:21:38.017 INFO [2] 459d711a-572f-4bec-bee0-901ef0300807 (d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.017 INFO [1] received activate with gen 1
Sep 22 23:21:38.018 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.018 INFO [1] client got ds_active_rx, promote! session 9ec08283-1762-4efd-adf7-8c5fe20cabd0
Sep 22 23:21:38.018 WARN [2] new RM replaced this: None
Sep 22 23:21:38.018 INFO listening, local_addr: 127.0.0.1:52052, task: repair
Sep 22 23:21:38.018 INFO [2] Starts reconcile loop
Sep 22 23:21:38.018 INFO [2] received activate with gen 1
Sep 22 23:21:38.018 INFO [2] client got ds_active_rx, promote! session 9ec08283-1762-4efd-adf7-8c5fe20cabd0
Sep 22 23:21:38.018 INFO [0] 127.0.0.1:44460 task reports connection:true
Sep 22 23:21:38.018 INFO 459d711a-572f-4bec-bee0-901ef0300807 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.018 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.018 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.018 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.018 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.018 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.018 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.018 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.018 INFO UpstairsConnection { upstairs_id: 7aa67697-0340-40b6-98b6-fb2ed6103ed6, session_id: 9ec08283-1762-4efd-adf7-8c5fe20cabd0, gen: 1 } is now active (read-write)
Sep 22 23:21:38.018 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.018 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.018 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52052, task: repair
Sep 22 23:21:38.018 INFO Max found gen is 1
Sep 22 23:21:38.018 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.018 INFO Next flush: 1
Sep 22 23:21:38.018 INFO Using repair address: 127.0.0.1:52052, task: main
Sep 22 23:21:38.018 INFO All extents match
Sep 22 23:21:38.018 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.018 INFO No downstairs repair required
Sep 22 23:21:38.018 INFO UpstairsConnection { upstairs_id: 7aa67697-0340-40b6-98b6-fb2ed6103ed6, session_id: 9ec08283-1762-4efd-adf7-8c5fe20cabd0, gen: 1 } is now active (read-write)
Sep 22 23:21:38.018 INFO No initial repair work was required
Sep 22 23:21:38.018 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.018 INFO 459d711a-572f-4bec-bee0-901ef0300807 is now active with session: d2b2fd2b-930d-4a52-b26d-41e04dc4ed1d
Sep 22 23:21:38.018 INFO 459d711a-572f-4bec-bee0-901ef0300807 Set Active after no repair
Sep 22 23:21:38.018 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.018 INFO Set check for repair
Sep 22 23:21:38.018 INFO UpstairsConnection { upstairs_id: 7aa67697-0340-40b6-98b6-fb2ed6103ed6, session_id: 9ec08283-1762-4efd-adf7-8c5fe20cabd0, gen: 1 } is now active (read-write)
Sep 22 23:21:38.018 INFO [1] 127.0.0.1:32949 task reports connection:true
Sep 22 23:21:38.018 INFO 459d711a-572f-4bec-bee0-901ef0300807 Active Active Active
Sep 22 23:21:38.018 INFO Set check for repair
Sep 22 23:21:38.018 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.018 INFO [2] 127.0.0.1:53630 task reports connection:true
Sep 22 23:21:38.018 INFO 459d711a-572f-4bec-bee0-901ef0300807 Active Active Active
Sep 22 23:21:38.018 INFO Set check for repair
Sep 22 23:21:38.018 INFO [0] received reconcile message
Sep 22 23:21:38.018 INFO [0] All repairs completed, exit
Sep 22 23:21:38.018 INFO Created new region file "/tmp/downstairs-sqEOUB0C/region.json"
Sep 22 23:21:38.018 INFO [0] Starts cmd_loop
Sep 22 23:21:38.018 INFO [0] downstairs client at 127.0.0.1:64852 has UUID fe4a84c6-9d76-42b5-9360-3287766e9e73
Sep 22 23:21:38.018 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: fe4a84c6-9d76-42b5-9360-3287766e9e73, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.018 INFO [1] received reconcile message
Sep 22 23:21:38.018 INFO [1] All repairs completed, exit
Sep 22 23:21:38.018 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 WaitActive WaitActive WaitActive
Sep 22 23:21:38.018 INFO [1] Starts cmd_loop
Sep 22 23:21:38.018 INFO [2] received reconcile message
Sep 22 23:21:38.018 INFO [1] downstairs client at 127.0.0.1:55577 has UUID 2be75e90-0f81-4f50-bf88-09ce34abc850
Sep 22 23:21:38.018 INFO [2] All repairs completed, exit
Sep 22 23:21:38.018 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2be75e90-0f81-4f50-bf88-09ce34abc850, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.018 INFO [2] Starts cmd_loop
Sep 22 23:21:38.018 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 WaitActive WaitActive WaitActive
The guest has finished waiting for activation
Sep 22 23:21:38.018 INFO [2] downstairs client at 127.0.0.1:49689 has UUID 5366c239-e0fc-41ac-ab9f-b458e8730135
Sep 22 23:21:38.018 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 5366c239-e0fc-41ac-ab9f-b458e8730135, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.018 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 WaitActive WaitActive WaitActive
The guest has requested activation
Sep 22 23:21:38.019 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.019 INFO 57a23afd-0f7e-4863-905a-9c435a295364 active request set
Sep 22 23:21:38.019 INFO [0] received activate with gen 1
Sep 22 23:21:38.019 INFO [0] client got ds_active_rx, promote! session 465019ca-32d3-499c-bb90-0f856bcdf646
Sep 22 23:21:38.019 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.019 INFO [1] received activate with gen 1
Sep 22 23:21:38.019 INFO [1] client got ds_active_rx, promote! session 465019ca-32d3-499c-bb90-0f856bcdf646
Sep 22 23:21:38.019 INFO [2] received activate with gen 1
Sep 22 23:21:38.019 INFO [2] client got ds_active_rx, promote! session 465019ca-32d3-499c-bb90-0f856bcdf646
Sep 22 23:21:38.019 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.019 INFO UpstairsConnection { upstairs_id: 57a23afd-0f7e-4863-905a-9c435a295364, session_id: 465019ca-32d3-499c-bb90-0f856bcdf646, gen: 1 } is now active (read-write)
Sep 22 23:21:38.019 INFO UpstairsConnection { upstairs_id: 57a23afd-0f7e-4863-905a-9c435a295364, session_id: 465019ca-32d3-499c-bb90-0f856bcdf646, gen: 1 } is now active (read-write)
Sep 22 23:21:38.019 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.019 INFO UpstairsConnection { upstairs_id: 57a23afd-0f7e-4863-905a-9c435a295364, session_id: 465019ca-32d3-499c-bb90-0f856bcdf646, gen: 1 } is now active (read-write)
Sep 22 23:21:38.019 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.019 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.020 INFO [0] downstairs client at 127.0.0.1:62409 has UUID 3e5893e8-0146-44bc-9dcd-87f1c9321550
Sep 22 23:21:38.020 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3e5893e8-0146-44bc-9dcd-87f1c9321550, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.020 INFO 57a23afd-0f7e-4863-905a-9c435a295364 WaitActive WaitActive WaitActive
Sep 22 23:21:38.020 INFO [1] downstairs client at 127.0.0.1:58576 has UUID 1d55152f-d449-4fe0-aafd-0fc27bdcca9b
Sep 22 23:21:38.020 INFO [0] 7aa67697-0340-40b6-98b6-fb2ed6103ed6 (9ec08283-1762-4efd-adf7-8c5fe20cabd0) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.020 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1d55152f-d449-4fe0-aafd-0fc27bdcca9b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.020 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.020 WARN [0] new RM replaced this: None
Sep 22 23:21:38.020 INFO 57a23afd-0f7e-4863-905a-9c435a295364 WaitActive WaitActive WaitActive
Sep 22 23:21:38.020 INFO [0] Starts reconcile loop
Sep 22 23:21:38.020 INFO [2] downstairs client at 127.0.0.1:55350 has UUID 1839dd14-6901-4a5d-8bee-dbdeaa422af1
Sep 22 23:21:38.020 INFO [1] 7aa67697-0340-40b6-98b6-fb2ed6103ed6 (9ec08283-1762-4efd-adf7-8c5fe20cabd0) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.020 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.020 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1839dd14-6901-4a5d-8bee-dbdeaa422af1, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.020 WARN [1] new RM replaced this: None
Sep 22 23:21:38.020 INFO [1] Starts reconcile loop
Sep 22 23:21:38.020 INFO 57a23afd-0f7e-4863-905a-9c435a295364 WaitActive WaitActive WaitActive
Sep 22 23:21:38.020 INFO [2] 7aa67697-0340-40b6-98b6-fb2ed6103ed6 (9ec08283-1762-4efd-adf7-8c5fe20cabd0) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.020 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.020 WARN [2] new RM replaced this: None
Sep 22 23:21:38.020 INFO [2] Starts reconcile loop
Sep 22 23:21:38.020 INFO [0] 127.0.0.1:64852 task reports connection:true
Sep 22 23:21:38.020 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.020 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.020 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.020 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.020 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.020 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.020 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.020 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.020 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.020 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.020 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.020 INFO Max found gen is 1
Sep 22 23:21:38.020 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.020 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.020 INFO Next flush: 1
Sep 22 23:21:38.020 INFO All extents match
Sep 22 23:21:38.020 INFO No downstairs repair required
Sep 22 23:21:38.020 INFO No initial repair work was required
Sep 22 23:21:38.020 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.020 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 is now active with session: 9ec08283-1762-4efd-adf7-8c5fe20cabd0
Sep 22 23:21:38.020 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 Set Active after no repair
Sep 22 23:21:38.020 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.020 INFO Set check for repair
Sep 22 23:21:38.020 INFO [1] 127.0.0.1:55577 task reports connection:true
Sep 22 23:21:38.020 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 Active Active Active
Sep 22 23:21:38.020 INFO Set check for repair
Sep 22 23:21:38.020 INFO [2] 127.0.0.1:49689 task reports connection:true
Sep 22 23:21:38.020 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 Active Active Active
Sep 22 23:21:38.020 INFO Set check for repair
Sep 22 23:21:38.020 INFO [0] received reconcile message
Sep 22 23:21:38.020 INFO [0] All repairs completed, exit
Sep 22 23:21:38.020 INFO [0] Starts cmd_loop
Sep 22 23:21:38.021 INFO [1] received reconcile message
Sep 22 23:21:38.021 INFO [1] All repairs completed, exit
Sep 22 23:21:38.021 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.021 INFO [1] Starts cmd_loop
Sep 22 23:21:38.021 INFO [2] received reconcile message
Sep 22 23:21:38.021 INFO [2] All repairs completed, exit
Sep 22 23:21:38.021 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.021 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.021 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.021 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.021 INFO Created new region file "/tmp/downstairs-wXyx2JF2/region.json"
Sep 22 23:21:38.021 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.021 INFO [0] 57a23afd-0f7e-4863-905a-9c435a295364 (465019ca-32d3-499c-bb90-0f856bcdf646) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.021 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.021 WARN [0] new RM replaced this: None
Sep 22 23:21:38.021 INFO [0] Starts reconcile loop
Sep 22 23:21:38.021 INFO [1] 57a23afd-0f7e-4863-905a-9c435a295364 (465019ca-32d3-499c-bb90-0f856bcdf646) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.021 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.021 WARN [1] new RM replaced this: None
Sep 22 23:21:38.021 INFO [1] Starts reconcile loop
Sep 22 23:21:38.021 INFO [2] 57a23afd-0f7e-4863-905a-9c435a295364 (465019ca-32d3-499c-bb90-0f856bcdf646) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.021 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.021 WARN [2] new RM replaced this: None
Sep 22 23:21:38.021 INFO [2] Starts reconcile loop
Sep 22 23:21:38.021 INFO [0] 127.0.0.1:62409 task reports connection:true
Sep 22 23:21:38.021 INFO 57a23afd-0f7e-4863-905a-9c435a295364 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.022 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.022 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.022 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.022 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.022 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.022 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.022 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.022 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.022 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.022 INFO Max found gen is 1
Sep 22 23:21:38.022 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.022 INFO Next flush: 1
Sep 22 23:21:38.022 INFO All extents match
Sep 22 23:21:38.022 INFO No downstairs repair required
Sep 22 23:21:38.022 INFO No initial repair work was required
Sep 22 23:21:38.022 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.022 INFO 57a23afd-0f7e-4863-905a-9c435a295364 is now active with session: 465019ca-32d3-499c-bb90-0f856bcdf646
Sep 22 23:21:38.022 INFO 57a23afd-0f7e-4863-905a-9c435a295364 Set Active after no repair
Sep 22 23:21:38.022 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.022 INFO Set check for repair
Sep 22 23:21:38.022 INFO [1] 127.0.0.1:58576 task reports connection:true
Sep 22 23:21:38.022 INFO 57a23afd-0f7e-4863-905a-9c435a295364 Active Active Active
Sep 22 23:21:38.022 INFO Set check for repair
Sep 22 23:21:38.022 INFO [2] 127.0.0.1:55350 task reports connection:true
Sep 22 23:21:38.022 INFO 57a23afd-0f7e-4863-905a-9c435a295364 Active Active Active
Sep 22 23:21:38.022 INFO Set check for repair
Sep 22 23:21:38.022 INFO [0] received reconcile message
Sep 22 23:21:38.022 INFO [0] All repairs completed, exit
Sep 22 23:21:38.022 INFO [0] Starts cmd_loop
Sep 22 23:21:38.022 INFO [1] received reconcile message
Sep 22 23:21:38.022 INFO [1] All repairs completed, exit
Sep 22 23:21:38.022 INFO [1] Starts cmd_loop
Sep 22 23:21:38.022 INFO [2] received reconcile message
Sep 22 23:21:38.022 INFO [2] All repairs completed, exit
Sep 22 23:21:38.022 INFO [2] Starts cmd_loop
Sep 22 23:21:38.022 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.022 INFO Opened existing region file "/tmp/downstairs-sqEOUB0C/region.json"
Sep 22 23:21:38.022 INFO Database read version 1
The guest has finished waiting for activation
Sep 22 23:21:38.022 INFO Database write version 1
Sep 22 23:21:38.023 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.023 DEBG up_ds_listen was notified
Sep 22 23:21:38.023 DEBG up_ds_listen process 1000
Sep 22 23:21:38.023 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.023 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.024 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.024 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.025 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.025 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.025 DEBG up_ds_listen was notified
Sep 22 23:21:38.025 DEBG up_ds_listen process 1000
Sep 22 23:21:38.025 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.025 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.025 INFO UUID: 86a54504-22a8-4d69-94ec-02ec4287a819
Sep 22 23:21:38.025 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.026 INFO Opened existing region file "/tmp/downstairs-wXyx2JF2/region.json"
Sep 22 23:21:38.026 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.026 INFO Database read version 1
Sep 22 23:21:38.026 INFO Database write version 1
Sep 22 23:21:38.026 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.026 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.026 INFO Using address: 127.0.0.1:56772, task: main
Sep 22 23:21:38.026 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.026 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:64690, task: repair
Sep 22 23:21:38.026 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:64690, task: repair
Sep 22 23:21:38.026 INFO listening, local_addr: 127.0.0.1:64690, task: repair
Sep 22 23:21:38.026 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:64690, task: repair
Sep 22 23:21:38.026 INFO Using repair address: 127.0.0.1:64690, task: main
Sep 22 23:21:38.026 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.027 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.027 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.027 INFO Created new region file "/tmp/downstairs-jWkf3rfF/region.json"
Sep 22 23:21:38.027 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.027 INFO UUID: f4ab8931-de6c-4224-afb4-a8e9cc1a1136
Sep 22 23:21:38.027 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.027 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.027 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.027 INFO Using address: 127.0.0.1:55826, task: main
Sep 22 23:21:38.028 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.028 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.028 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:63059, task: repair
Sep 22 23:21:38.028 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:63059, task: repair
Sep 22 23:21:38.028 INFO listening, local_addr: 127.0.0.1:63059, task: repair
Sep 22 23:21:38.028 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:21:38.028 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:63059, task: repair
Sep 22 23:21:38.028 INFO Using repair address: 127.0.0.1:63059, task: main
Sep 22 23:21:38.028 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.029 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.029 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.029 WARN 7aa67697-0340-40b6-98b6-fb2ed6103ed6 request to replace downstairs 127.0.0.1:64852 with 127.0.0.1:55826
Sep 22 23:21:38.029 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 found old target: 127.0.0.1:64852 at 0
Sep 22 23:21:38.029 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 replacing old: 127.0.0.1:64852 at 0
Sep 22 23:21:38.029 INFO [0] client skip 0 in process jobs because fault, : downstairs
Sep 22 23:21:38.029 INFO [0] changed 0 jobs to fault skipped, : downstairs
Sep 22 23:21:38.029 INFO [0] 7aa67697-0340-40b6-98b6-fb2ed6103ed6 (9ec08283-1762-4efd-adf7-8c5fe20cabd0) Active Active Active ds_transition to Replacing
Sep 22 23:21:38.029 INFO [0] Transition from Active to Replacing
Sep 22 23:21:38.029 WARN 7aa67697-0340-40b6-98b6-fb2ed6103ed6 request to replace downstairs 127.0.0.1:64852 with 127.0.0.1:55826
Sep 22 23:21:38.029 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.029 INFO 7aa67697-0340-40b6-98b6-fb2ed6103ed6 found new target: 127.0.0.1:55826 at 0
Sep 22 23:21:38.029 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.030 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.030 INFO Opened existing region file "/tmp/downstairs-jWkf3rfF/region.json"
Sep 22 23:21:38.030 INFO Database read version 1
Sep 22 23:21:38.030 INFO Database write version 1
Sep 22 23:21:38.030 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:38.030 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.031 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.031 DEBG up_ds_listen was notified
Sep 22 23:21:38.031 DEBG up_ds_listen process 1001
Sep 22 23:21:38.031 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.031 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.032 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:21:38.032 INFO UUID: f98632f4-9734-43e6-b572-d9debb4ad27f
Sep 22 23:21:38.032 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.032 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.032 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.032 INFO Using address: 127.0.0.1:62775, task: main
Sep 22 23:21:38.032 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.033 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.033 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50255, task: repair
Sep 22 23:21:38.033 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.033 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50255, task: repair
Sep 22 23:21:38.033 INFO listening, local_addr: 127.0.0.1:50255, task: repair
Sep 22 23:21:38.033 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.033 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50255, task: repair
Sep 22 23:21:38.033 INFO Using repair address: 127.0.0.1:50255, task: main
Sep 22 23:21:38.033 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.034 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:38.034 DEBG [1] Read already AckReady 1001, : downstairs
test test::integration_test_volume_twice_replace_downstairs ... ok
Sep 22 23:21:38.034 INFO Upstairs starts
Sep 22 23:21:38.034 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.034 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.034 INFO Crucible stats registered with UUID: 252f3746-7497-4f5d-930d-d4a80969a0ed
Sep 22 23:21:38.034 INFO Crucible 252f3746-7497-4f5d-930d-d4a80969a0ed has session id: 4e25679f-917a-45ca-9f0e-f27058713d27
Sep 22 23:21:38.034 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.034 INFO Created new region file "/tmp/downstairs-rsuZzCsU/region.json"
Sep 22 23:21:38.034 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.034 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.035 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.035 DEBG up_ds_listen was notified
Sep 22 23:21:38.035 DEBG up_ds_listen process 1001
Sep 22 23:21:38.035 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.035 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.035 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.035 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.035 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.035 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.035 INFO [0] connecting to 127.0.0.1:49721, looper: 0
Sep 22 23:21:38.035 INFO [1] connecting to 127.0.0.1:49229, looper: 1
Sep 22 23:21:38.035 INFO [2] connecting to 127.0.0.1:59698, looper: 2
Sep 22 23:21:38.035 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.035 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.035 INFO Flush timeout: 0.5
Sep 22 23:21:38.035 INFO accepted connection from 127.0.0.1:50277, task: main
Sep 22 23:21:38.036 INFO accepted connection from 127.0.0.1:48202, task: main
Sep 22 23:21:38.036 DEBG IO Write 1002 has deps []
Sep 22 23:21:38.036 INFO accepted connection from 127.0.0.1:50028, task: main
Sep 22 23:21:38.036 DEBG up_ds_listen was notified
Sep 22 23:21:38.036 DEBG up_ds_listen process 1002
Sep 22 23:21:38.036 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.036 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.036 INFO [0] 252f3746-7497-4f5d-930d-d4a80969a0ed looper connected, looper: 0
Sep 22 23:21:38.036 INFO [0] Proc runs for 127.0.0.1:49721 in state New
Sep 22 23:21:38.036 INFO [1] 252f3746-7497-4f5d-930d-d4a80969a0ed looper connected, looper: 1
Sep 22 23:21:38.036 INFO [1] Proc runs for 127.0.0.1:49229 in state New
Sep 22 23:21:38.036 INFO [2] 252f3746-7497-4f5d-930d-d4a80969a0ed looper connected, looper: 2
Sep 22 23:21:38.036 INFO [2] Proc runs for 127.0.0.1:59698 in state New
Sep 22 23:21:38.036 INFO Scrub check for f70a61ee-a9f0-49ce-8bae-d3843b954927
Sep 22 23:21:38.036 INFO Scrub for f70a61ee-a9f0-49ce-8bae-d3843b954927 begins
Sep 22 23:21:38.036 INFO Scrub with total_size:5120 block_size:512
Sep 22 23:21:38.036 INFO Scrubs from block 0 to 10 in (256) 131072 size IOs pm:0
Sep 22 23:21:38.036 INFO Adjust block_count to 10 at offset 0
Sep 22 23:21:38.037 INFO Upstairs starts
Sep 22 23:21:38.037 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.037 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.037 INFO Crucible stats registered with UUID: 723c92d6-2550-4896-aeb0-c3ba0579c262
Sep 22 23:21:38.037 INFO Crucible 723c92d6-2550-4896-aeb0-c3ba0579c262 has session id: 0b68b8af-adb3-4d45-8261-82fa9b42ae96
Sep 22 23:21:38.037 INFO Connection request from 252f3746-7497-4f5d-930d-d4a80969a0ed with version 4, task: proc
Sep 22 23:21:38.037 INFO upstairs UpstairsConnection { upstairs_id: 252f3746-7497-4f5d-930d-d4a80969a0ed, session_id: 2f207150-16ea-4b9b-9a7d-235b5da77cb3, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.037 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.037 INFO Connection request from 252f3746-7497-4f5d-930d-d4a80969a0ed with version 4, task: proc
Sep 22 23:21:38.037 INFO Opened existing region file "/tmp/downstairs-rsuZzCsU/region.json"
Sep 22 23:21:38.037 INFO upstairs UpstairsConnection { upstairs_id: 252f3746-7497-4f5d-930d-d4a80969a0ed, session_id: 2f207150-16ea-4b9b-9a7d-235b5da77cb3, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.037 INFO Database read version 1
Sep 22 23:21:38.037 INFO Database write version 1
Sep 22 23:21:38.037 INFO Connection request from 252f3746-7497-4f5d-930d-d4a80969a0ed with version 4, task: proc
Sep 22 23:21:38.037 INFO upstairs UpstairsConnection { upstairs_id: 252f3746-7497-4f5d-930d-d4a80969a0ed, session_id: 2f207150-16ea-4b9b-9a7d-235b5da77cb3, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.037 INFO [0] connecting to 127.0.0.1:60338, looper: 0
Sep 22 23:21:38.037 INFO [1] connecting to 127.0.0.1:56772, looper: 1
Sep 22 23:21:38.038 INFO [2] connecting to 127.0.0.1:62775, looper: 2
Sep 22 23:21:38.038 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.038 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.038 INFO Flush timeout: 0.5
Sep 22 23:21:38.038 INFO accepted connection from 127.0.0.1:48111, task: main
Sep 22 23:21:38.038 INFO [0] 252f3746-7497-4f5d-930d-d4a80969a0ed (2f207150-16ea-4b9b-9a7d-235b5da77cb3) New New New ds_transition to WaitActive
Sep 22 23:21:38.038 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.038 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.038 INFO [1] 252f3746-7497-4f5d-930d-d4a80969a0ed (2f207150-16ea-4b9b-9a7d-235b5da77cb3) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.038 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.038 INFO accepted connection from 127.0.0.1:33070, task: main
Sep 22 23:21:38.038 INFO [2] 252f3746-7497-4f5d-930d-d4a80969a0ed (2f207150-16ea-4b9b-9a7d-235b5da77cb3) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.039 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.039 INFO accepted connection from 127.0.0.1:56574, task: main
Sep 22 23:21:38.039 INFO [1] 723c92d6-2550-4896-aeb0-c3ba0579c262 looper connected, looper: 1
Sep 22 23:21:38.039 INFO [1] Proc runs for 127.0.0.1:56772 in state New
Sep 22 23:21:38.039 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.039 INFO [0] 723c92d6-2550-4896-aeb0-c3ba0579c262 looper connected, looper: 0
Sep 22 23:21:38.039 INFO [0] Proc runs for 127.0.0.1:60338 in state New
Sep 22 23:21:38.039 INFO [2] 723c92d6-2550-4896-aeb0-c3ba0579c262 looper connected, looper: 2
Sep 22 23:21:38.039 INFO [2] Proc runs for 127.0.0.1:62775 in state New
Sep 22 23:21:38.039 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.039 INFO UUID: f00b2386-14a7-47f7-ad70-b49b4e8545ea
Sep 22 23:21:38.039 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.040 INFO Connection request from 723c92d6-2550-4896-aeb0-c3ba0579c262 with version 4, task: proc
Sep 22 23:21:38.040 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.040 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.040 INFO upstairs UpstairsConnection { upstairs_id: 723c92d6-2550-4896-aeb0-c3ba0579c262, session_id: 23df1fb4-419d-47ca-b93d-564d049b0bdc, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.040 INFO Using address: 127.0.0.1:48412, task: main
Sep 22 23:21:38.040 INFO Connection request from 723c92d6-2550-4896-aeb0-c3ba0579c262 with version 4, task: proc
Sep 22 23:21:38.040 INFO upstairs UpstairsConnection { upstairs_id: 723c92d6-2550-4896-aeb0-c3ba0579c262, session_id: 23df1fb4-419d-47ca-b93d-564d049b0bdc, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.040 INFO Connection request from 723c92d6-2550-4896-aeb0-c3ba0579c262 with version 4, task: proc
Sep 22 23:21:38.040 INFO upstairs UpstairsConnection { upstairs_id: 723c92d6-2550-4896-aeb0-c3ba0579c262, session_id: 23df1fb4-419d-47ca-b93d-564d049b0bdc, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.040 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.040 INFO [1] 723c92d6-2550-4896-aeb0-c3ba0579c262 (23df1fb4-419d-47ca-b93d-564d049b0bdc) New New New ds_transition to WaitActive
Sep 22 23:21:38.040 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.040 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60475, task: repair
Sep 22 23:21:38.040 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60475, task: repair
Sep 22 23:21:38.040 INFO [0] 723c92d6-2550-4896-aeb0-c3ba0579c262 (23df1fb4-419d-47ca-b93d-564d049b0bdc) New WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.040 INFO listening, local_addr: 127.0.0.1:60475, task: repair
Sep 22 23:21:38.040 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.040 INFO [2] 723c92d6-2550-4896-aeb0-c3ba0579c262 (23df1fb4-419d-47ca-b93d-564d049b0bdc) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.040 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.040 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60475, task: repair
Sep 22 23:21:38.040 INFO Using repair address: 127.0.0.1:60475, task: main
Sep 22 23:21:38.040 INFO No SSL acceptor configured, task: main
The guest has requested activation
Sep 22 23:21:38.041 INFO 252f3746-7497-4f5d-930d-d4a80969a0ed active request set
Sep 22 23:21:38.041 INFO [0] received activate with gen 1
Sep 22 23:21:38.041 INFO [0] client got ds_active_rx, promote! session 2f207150-16ea-4b9b-9a7d-235b5da77cb3
Sep 22 23:21:38.041 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.041 INFO [1] received activate with gen 1
Sep 22 23:21:38.041 INFO [1] client got ds_active_rx, promote! session 2f207150-16ea-4b9b-9a7d-235b5da77cb3
Sep 22 23:21:38.041 INFO Created new region file "/tmp/downstairs-6LcCIRlu/region.json"
Sep 22 23:21:38.041 INFO [2] received activate with gen 1
Sep 22 23:21:38.041 INFO [2] client got ds_active_rx, promote! session 2f207150-16ea-4b9b-9a7d-235b5da77cb3
Sep 22 23:21:38.041 INFO UpstairsConnection { upstairs_id: 252f3746-7497-4f5d-930d-d4a80969a0ed, session_id: 2f207150-16ea-4b9b-9a7d-235b5da77cb3, gen: 1 } is now active (read-write)
Sep 22 23:21:38.041 INFO UpstairsConnection { upstairs_id: 252f3746-7497-4f5d-930d-d4a80969a0ed, session_id: 2f207150-16ea-4b9b-9a7d-235b5da77cb3, gen: 1 } is now active (read-write)
Sep 22 23:21:38.041 INFO UpstairsConnection { upstairs_id: 252f3746-7497-4f5d-930d-d4a80969a0ed, session_id: 2f207150-16ea-4b9b-9a7d-235b5da77cb3, gen: 1 } is now active (read-write)
Sep 22 23:21:38.042 INFO [0] downstairs client at 127.0.0.1:49721 has UUID 5e565886-9774-4b9d-9633-93979cb0f8ae
Sep 22 23:21:38.042 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 5e565886-9774-4b9d-9633-93979cb0f8ae, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.042 INFO 252f3746-7497-4f5d-930d-d4a80969a0ed WaitActive WaitActive WaitActive
Sep 22 23:21:38.042 INFO [1] downstairs client at 127.0.0.1:49229 has UUID 6c7e6774-c872-46de-b630-fdc717de2268
Sep 22 23:21:38.042 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6c7e6774-c872-46de-b630-fdc717de2268, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.042 INFO 252f3746-7497-4f5d-930d-d4a80969a0ed WaitActive WaitActive WaitActive
Sep 22 23:21:38.042 INFO [2] downstairs client at 127.0.0.1:59698 has UUID 709bf894-ad93-42e1-b7a3-127d2f1f8b04
Sep 22 23:21:38.042 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 709bf894-ad93-42e1-b7a3-127d2f1f8b04, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.042 DEBG IO Write 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:21:38.042 INFO 252f3746-7497-4f5d-930d-d4a80969a0ed WaitActive WaitActive WaitActive
Sep 22 23:21:38.042 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.043 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.043 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.043 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.044 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.044 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.044 INFO [0] 252f3746-7497-4f5d-930d-d4a80969a0ed (2f207150-16ea-4b9b-9a7d-235b5da77cb3) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.044 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.044 WARN [0] new RM replaced this: None
Sep 22 23:21:38.044 INFO [0] Starts reconcile loop
Sep 22 23:21:38.044 INFO [1] 252f3746-7497-4f5d-930d-d4a80969a0ed (2f207150-16ea-4b9b-9a7d-235b5da77cb3) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.044 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.044 WARN [1] new RM replaced this: None
Sep 22 23:21:38.044 INFO [1] Starts reconcile loop
Sep 22 23:21:38.044 INFO [2] 252f3746-7497-4f5d-930d-d4a80969a0ed (2f207150-16ea-4b9b-9a7d-235b5da77cb3) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.044 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.044 WARN [2] new RM replaced this: None
Sep 22 23:21:38.044 INFO [2] Starts reconcile loop
Sep 22 23:21:38.044 INFO [0] 127.0.0.1:49721 task reports connection:true
Sep 22 23:21:38.044 INFO 252f3746-7497-4f5d-930d-d4a80969a0ed WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.044 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.044 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.044 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.045 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.045 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.045 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.045 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.045 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.045 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.045 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.045 INFO Opened existing region file "/tmp/downstairs-6LcCIRlu/region.json"
Sep 22 23:21:38.045 INFO Max found gen is 1
Sep 22 23:21:38.045 INFO Database read version 1
Sep 22 23:21:38.045 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.045 INFO Database write version 1
Sep 22 23:21:38.045 INFO Next flush: 1
Sep 22 23:21:38.045 INFO All extents match
Sep 22 23:21:38.045 INFO No downstairs repair required
Sep 22 23:21:38.045 INFO No initial repair work was required
Sep 22 23:21:38.045 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.045 INFO 252f3746-7497-4f5d-930d-d4a80969a0ed is now active with session: 2f207150-16ea-4b9b-9a7d-235b5da77cb3
Sep 22 23:21:38.045 INFO 252f3746-7497-4f5d-930d-d4a80969a0ed Set Active after no repair
Sep 22 23:21:38.045 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.045 INFO Set check for repair
Sep 22 23:21:38.045 INFO [1] 127.0.0.1:49229 task reports connection:true
Sep 22 23:21:38.045 INFO 252f3746-7497-4f5d-930d-d4a80969a0ed Active Active Active
Sep 22 23:21:38.045 INFO Set check for repair
Sep 22 23:21:38.045 INFO [2] 127.0.0.1:59698 task reports connection:true
Sep 22 23:21:38.045 INFO 252f3746-7497-4f5d-930d-d4a80969a0ed Active Active Active
Sep 22 23:21:38.045 INFO Set check for repair
Sep 22 23:21:38.045 INFO [0] received reconcile message
Sep 22 23:21:38.045 INFO [0] All repairs completed, exit
Sep 22 23:21:38.045 INFO [0] Starts cmd_loop
Sep 22 23:21:38.045 INFO [1] received reconcile message
Sep 22 23:21:38.045 INFO [1] All repairs completed, exit
Sep 22 23:21:38.045 INFO [1] Starts cmd_loop
Sep 22 23:21:38.045 INFO [2] received reconcile message
Sep 22 23:21:38.045 INFO [2] All repairs completed, exit
Sep 22 23:21:38.045 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
The guest has requested activation
Sep 22 23:21:38.046 INFO 723c92d6-2550-4896-aeb0-c3ba0579c262 active request set
Sep 22 23:21:38.046 INFO [0] received activate with gen 1
Sep 22 23:21:38.046 INFO [0] client got ds_active_rx, promote! session 23df1fb4-419d-47ca-b93d-564d049b0bdc
Sep 22 23:21:38.046 INFO [1] received activate with gen 1
Sep 22 23:21:38.046 INFO [1] client got ds_active_rx, promote! session 23df1fb4-419d-47ca-b93d-564d049b0bdc
Sep 22 23:21:38.046 INFO [2] received activate with gen 1
Sep 22 23:21:38.046 INFO [2] client got ds_active_rx, promote! session 23df1fb4-419d-47ca-b93d-564d049b0bdc
Sep 22 23:21:38.046 DEBG up_ds_listen was notified
Sep 22 23:21:38.046 DEBG up_ds_listen process 1003
Sep 22 23:21:38.046 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:38.046 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.046 INFO UpstairsConnection { upstairs_id: 723c92d6-2550-4896-aeb0-c3ba0579c262, session_id: 23df1fb4-419d-47ca-b93d-564d049b0bdc, gen: 1 } is now active (read-write)
Sep 22 23:21:38.046 INFO Scrub at offset 10/10 sp:10
Sep 22 23:21:38.046 INFO UpstairsConnection { upstairs_id: 723c92d6-2550-4896-aeb0-c3ba0579c262, session_id: 23df1fb4-419d-47ca-b93d-564d049b0bdc, gen: 1 } is now active (read-write)
Sep 22 23:21:38.047 INFO UpstairsConnection { upstairs_id: 723c92d6-2550-4896-aeb0-c3ba0579c262, session_id: 23df1fb4-419d-47ca-b93d-564d049b0bdc, gen: 1 } is now active (read-write)
Sep 22 23:21:38.047 INFO UUID: c47df1e6-d581-419f-94ff-178a0e67a3ed
Sep 22 23:21:38.047 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.047 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.047 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.047 INFO Using address: 127.0.0.1:41943, task: main
Sep 22 23:21:38.047 INFO [1] downstairs client at 127.0.0.1:56772 has UUID 86a54504-22a8-4d69-94ec-02ec4287a819
Sep 22 23:21:38.047 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 86a54504-22a8-4d69-94ec-02ec4287a819, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.047 INFO 723c92d6-2550-4896-aeb0-c3ba0579c262 WaitActive WaitActive WaitActive
Sep 22 23:21:38.047 INFO [0] downstairs client at 127.0.0.1:60338 has UUID 137bc727-67de-4ee4-a302-cef52368a28d
Sep 22 23:21:38.047 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 137bc727-67de-4ee4-a302-cef52368a28d, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.047 INFO 723c92d6-2550-4896-aeb0-c3ba0579c262 WaitActive WaitActive WaitActive
Sep 22 23:21:38.047 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.047 INFO [2] downstairs client at 127.0.0.1:62775 has UUID f98632f4-9734-43e6-b572-d9debb4ad27f
Sep 22 23:21:38.047 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f98632f4-9734-43e6-b572-d9debb4ad27f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.047 INFO Scrub f70a61ee-a9f0-49ce-8bae-d3843b954927 done in 0 seconds. Retries:0 scrub_size:5120 size:10 pause_milli:0
Sep 22 23:21:38.048 INFO 723c92d6-2550-4896-aeb0-c3ba0579c262 WaitActive WaitActive WaitActive
Sep 22 23:21:38.047 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:34906, task: repair
Sep 22 23:21:38.048 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:34906, task: repair
Sep 22 23:21:38.048 DEBG IO Flush 1004 has deps [JobId(1003), JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:21:38.048 INFO listening, local_addr: 127.0.0.1:34906, task: repair
Sep 22 23:21:38.048 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.048 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:34906, task: repair
Sep 22 23:21:38.048 INFO Using repair address: 127.0.0.1:34906, task: main
Sep 22 23:21:38.048 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.048 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.048 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.048 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.048 INFO Created new region file "/tmp/downstairs-lwXKwwR7/region.json"
Sep 22 23:21:38.048 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.049 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.049 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.049 INFO [1] 723c92d6-2550-4896-aeb0-c3ba0579c262 (23df1fb4-419d-47ca-b93d-564d049b0bdc) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.049 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.049 WARN [1] new RM replaced this: None
Sep 22 23:21:38.049 INFO [1] Starts reconcile loop
Sep 22 23:21:38.049 INFO [0] 723c92d6-2550-4896-aeb0-c3ba0579c262 (23df1fb4-419d-47ca-b93d-564d049b0bdc) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.049 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.049 WARN [0] new RM replaced this: None
Sep 22 23:21:38.049 INFO [0] Starts reconcile loop
Sep 22 23:21:38.049 INFO [2] 723c92d6-2550-4896-aeb0-c3ba0579c262 (23df1fb4-419d-47ca-b93d-564d049b0bdc) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.049 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.049 WARN [2] new RM replaced this: None
Sep 22 23:21:38.049 INFO [2] Starts reconcile loop
Sep 22 23:21:38.049 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.049 INFO [1] 127.0.0.1:56772 task reports connection:true
Sep 22 23:21:38.049 INFO 723c92d6-2550-4896-aeb0-c3ba0579c262 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.049 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.049 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.049 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.049 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.049 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.049 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.050 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.050 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.050 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.050 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.050 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.050 INFO Max found gen is 1
Sep 22 23:21:38.050 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.050 INFO Next flush: 1
Sep 22 23:21:38.050 INFO All extents match
Sep 22 23:21:38.050 INFO No downstairs repair required
Sep 22 23:21:38.050 INFO No initial repair work was required
Sep 22 23:21:38.050 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.050 INFO 723c92d6-2550-4896-aeb0-c3ba0579c262 is now active with session: 23df1fb4-419d-47ca-b93d-564d049b0bdc
Sep 22 23:21:38.050 INFO 723c92d6-2550-4896-aeb0-c3ba0579c262 Set Active after no repair
Sep 22 23:21:38.050 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.050 INFO Set check for repair
Sep 22 23:21:38.050 DEBG up_ds_listen was notified
Sep 22 23:21:38.050 INFO [0] 127.0.0.1:60338 task reports connection:true
Sep 22 23:21:38.050 INFO 723c92d6-2550-4896-aeb0-c3ba0579c262 Active Active Active
Sep 22 23:21:38.050 DEBG up_ds_listen process 1004
Sep 22 23:21:38.050 INFO Set check for repair
Sep 22 23:21:38.050 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:21:38.050 INFO [2] 127.0.0.1:62775 task reports connection:true
Sep 22 23:21:38.050 INFO 723c92d6-2550-4896-aeb0-c3ba0579c262 Active Active Active
Sep 22 23:21:38.050 INFO Set check for repair
Sep 22 23:21:38.050 DEBG [rc] retire 1004 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:21:38.050 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.050 INFO [0] received reconcile message
Sep 22 23:21:38.050 INFO [0] All repairs completed, exit
Sep 22 23:21:38.050 INFO [0] Starts cmd_loop
Sep 22 23:21:38.050 INFO [1] received reconcile message
Sep 22 23:21:38.050 DEBG IO Flush 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:38.050 INFO [1] All repairs completed, exit
Sep 22 23:21:38.050 INFO [1] Starts cmd_loop
Sep 22 23:21:38.050 INFO [2] received reconcile message
Sep 22 23:21:38.050 INFO [2] All repairs completed, exit
Sep 22 23:21:38.050 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.051 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.051 DEBG up_ds_listen was notified
Sep 22 23:21:38.051 DEBG up_ds_listen process 1000
Sep 22 23:21:38.051 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.051 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.052 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.052 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.052 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.052 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.052 INFO Opened existing region file "/tmp/downstairs-lwXKwwR7/region.json"
Sep 22 23:21:38.052 INFO Database read version 1
Sep 22 23:21:38.052 INFO Database write version 1
Sep 22 23:21:38.052 DEBG up_ds_listen was notified
Sep 22 23:21:38.052 DEBG up_ds_listen process 1002
Sep 22 23:21:38.052 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.052 DEBG [rc] retire 1002 clears [JobId(1000), JobId(1001), JobId(1002)], : downstairs
Sep 22 23:21:38.052 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.052 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.053 DEBG IO Read  1005 has deps []
Sep 22 23:21:38.053 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.053 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.053 DEBG Read      :1005 deps:[] res:true
Sep 22 23:21:38.054 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.054 DEBG Read      :1005 deps:[] res:true
Sep 22 23:21:38.054 DEBG up_ds_listen was notified
Sep 22 23:21:38.054 DEBG up_ds_listen process 1000
Sep 22 23:21:38.054 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.054 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.054 INFO UUID: d170e9eb-99a7-40aa-8e4b-74aea23bd418
Sep 22 23:21:38.054 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.054 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.054 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.054 INFO Using address: 127.0.0.1:33214, task: main
Sep 22 23:21:38.054 DEBG Read      :1005 deps:[] res:true
Sep 22 23:21:38.055 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.055 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62619, task: repair
Sep 22 23:21:38.055 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62619, task: repair
Sep 22 23:21:38.055 INFO listening, local_addr: 127.0.0.1:62619, task: repair
Sep 22 23:21:38.055 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.055 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62619, task: repair
Sep 22 23:21:38.055 INFO Using repair address: 127.0.0.1:62619, task: main
Sep 22 23:21:38.055 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.055 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.056 INFO Upstairs starts
Sep 22 23:21:38.056 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.056 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.056 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.056 INFO Crucible stats registered with UUID: e4fe1d81-dc04-41e4-b745-9219b6982323
Sep 22 23:21:38.056 INFO Crucible e4fe1d81-dc04-41e4-b745-9219b6982323 has session id: b713c0d5-2ffc-4c94-a106-91b2d080c72b
Sep 22 23:21:38.056 INFO Connection request from d15c167b-c380-46a4-a953-864fc02495ed with version 4, task: proc
Sep 22 23:21:38.056 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.056 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:21:38.056 INFO upstairs UpstairsConnection { upstairs_id: d15c167b-c380-46a4-a953-864fc02495ed, session_id: d8b1013b-fbba-4bb2-a01a-1a978b72bafa, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.056 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.056 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.056 INFO Connection request from d15c167b-c380-46a4-a953-864fc02495ed with version 4, task: proc
Sep 22 23:21:38.056 INFO upstairs UpstairsConnection { upstairs_id: d15c167b-c380-46a4-a953-864fc02495ed, session_id: d8b1013b-fbba-4bb2-a01a-1a978b72bafa, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.056 INFO [0] connecting to 127.0.0.1:48412, looper: 0
Sep 22 23:21:38.056 INFO Connection request from d15c167b-c380-46a4-a953-864fc02495ed with version 4, task: proc
Sep 22 23:21:38.056 INFO upstairs UpstairsConnection { upstairs_id: d15c167b-c380-46a4-a953-864fc02495ed, session_id: d8b1013b-fbba-4bb2-a01a-1a978b72bafa, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.056 INFO [1] connecting to 127.0.0.1:41943, looper: 1
Sep 22 23:21:38.056 INFO [2] connecting to 127.0.0.1:33214, looper: 2
Sep 22 23:21:38.057 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.057 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.057 INFO Flush timeout: 0.5
Sep 22 23:21:38.057 INFO [0] d15c167b-c380-46a4-a953-864fc02495ed (d8b1013b-fbba-4bb2-a01a-1a978b72bafa) New New New ds_transition to WaitActive
Sep 22 23:21:38.057 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.057 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.057 INFO accepted connection from 127.0.0.1:50980, task: main
Sep 22 23:21:38.057 INFO [1] d15c167b-c380-46a4-a953-864fc02495ed (d8b1013b-fbba-4bb2-a01a-1a978b72bafa) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.057 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.057 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.057 INFO [2] d15c167b-c380-46a4-a953-864fc02495ed (d8b1013b-fbba-4bb2-a01a-1a978b72bafa) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.057 INFO accepted connection from 127.0.0.1:60993, task: main
Sep 22 23:21:38.057 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.057 INFO accepted connection from 127.0.0.1:39483, task: main
Sep 22 23:21:38.057 INFO [0] e4fe1d81-dc04-41e4-b745-9219b6982323 looper connected, looper: 0
Sep 22 23:21:38.057 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.057 INFO [0] Proc runs for 127.0.0.1:48412 in state New
Sep 22 23:21:38.057 INFO [1] e4fe1d81-dc04-41e4-b745-9219b6982323 looper connected, looper: 1
Sep 22 23:21:38.057 INFO [1] Proc runs for 127.0.0.1:41943 in state New
Sep 22 23:21:38.057 INFO [2] e4fe1d81-dc04-41e4-b745-9219b6982323 looper connected, looper: 2
Sep 22 23:21:38.057 INFO [2] Proc runs for 127.0.0.1:33214 in state New
Sep 22 23:21:38.058 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:38.058 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.058 INFO Connection request from e4fe1d81-dc04-41e4-b745-9219b6982323 with version 4, task: proc
Sep 22 23:21:38.058 INFO upstairs UpstairsConnection { upstairs_id: e4fe1d81-dc04-41e4-b745-9219b6982323, session_id: d2b38da3-e438-4e27-b550-ef7f093a76bf, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.058 INFO Connection request from e4fe1d81-dc04-41e4-b745-9219b6982323 with version 4, task: proc
Sep 22 23:21:38.058 INFO upstairs UpstairsConnection { upstairs_id: e4fe1d81-dc04-41e4-b745-9219b6982323, session_id: d2b38da3-e438-4e27-b550-ef7f093a76bf, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.058 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.058 INFO Connection request from e4fe1d81-dc04-41e4-b745-9219b6982323 with version 4, task: proc
Sep 22 23:21:38.058 DEBG up_ds_listen was notified
Sep 22 23:21:38.058 DEBG up_ds_listen process 1001
Sep 22 23:21:38.058 INFO upstairs UpstairsConnection { upstairs_id: e4fe1d81-dc04-41e4-b745-9219b6982323, session_id: d2b38da3-e438-4e27-b550-ef7f093a76bf, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.058 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.058 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.058 INFO [0] e4fe1d81-dc04-41e4-b745-9219b6982323 (d2b38da3-e438-4e27-b550-ef7f093a76bf) New New New ds_transition to WaitActive
The guest has requested activation
Sep 22 23:21:38.058 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.058 INFO d15c167b-c380-46a4-a953-864fc02495ed active request set
Sep 22 23:21:38.058 INFO [1] e4fe1d81-dc04-41e4-b745-9219b6982323 (d2b38da3-e438-4e27-b550-ef7f093a76bf) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.059 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.059 INFO [0] received activate with gen 1
Sep 22 23:21:38.059 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:21:38.059 INFO [0] client got ds_active_rx, promote! session d8b1013b-fbba-4bb2-a01a-1a978b72bafa
Sep 22 23:21:38.059 INFO [2] e4fe1d81-dc04-41e4-b745-9219b6982323 (d2b38da3-e438-4e27-b550-ef7f093a76bf) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.059 INFO [1] received activate with gen 1
Sep 22 23:21:38.059 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.059 INFO [1] client got ds_active_rx, promote! session d8b1013b-fbba-4bb2-a01a-1a978b72bafa
Sep 22 23:21:38.059 INFO [2] received activate with gen 1
Sep 22 23:21:38.059 DEBG [0] Read AckReady 1005, : downstairs
Sep 22 23:21:38.059 INFO [2] client got ds_active_rx, promote! session d8b1013b-fbba-4bb2-a01a-1a978b72bafa
The guest has requested activation
Sep 22 23:21:38.059 INFO UpstairsConnection { upstairs_id: d15c167b-c380-46a4-a953-864fc02495ed, session_id: d8b1013b-fbba-4bb2-a01a-1a978b72bafa, gen: 1 } is now active (read-write)
Sep 22 23:21:38.059 INFO e4fe1d81-dc04-41e4-b745-9219b6982323 active request set
Sep 22 23:21:38.059 INFO UpstairsConnection { upstairs_id: d15c167b-c380-46a4-a953-864fc02495ed, session_id: d8b1013b-fbba-4bb2-a01a-1a978b72bafa, gen: 1 } is now active (read-write)
Sep 22 23:21:38.059 INFO [0] received activate with gen 1
Sep 22 23:21:38.059 INFO [0] client got ds_active_rx, promote! session d2b38da3-e438-4e27-b550-ef7f093a76bf
Sep 22 23:21:38.059 INFO UpstairsConnection { upstairs_id: d15c167b-c380-46a4-a953-864fc02495ed, session_id: d8b1013b-fbba-4bb2-a01a-1a978b72bafa, gen: 1 } is now active (read-write)
Sep 22 23:21:38.059 INFO [1] received activate with gen 1
Sep 22 23:21:38.059 INFO [1] client got ds_active_rx, promote! session d2b38da3-e438-4e27-b550-ef7f093a76bf
Sep 22 23:21:38.059 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.059 INFO [2] received activate with gen 1
Sep 22 23:21:38.059 INFO [2] client got ds_active_rx, promote! session d2b38da3-e438-4e27-b550-ef7f093a76bf
Sep 22 23:21:38.059 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.059 INFO UpstairsConnection { upstairs_id: e4fe1d81-dc04-41e4-b745-9219b6982323, session_id: d2b38da3-e438-4e27-b550-ef7f093a76bf, gen: 1 } is now active (read-write)
Sep 22 23:21:38.059 INFO [0] downstairs client at 127.0.0.1:33968 has UUID d300473c-ae2a-4754-900f-58509b1b9961
Sep 22 23:21:38.059 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d300473c-ae2a-4754-900f-58509b1b9961, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.059 INFO UpstairsConnection { upstairs_id: e4fe1d81-dc04-41e4-b745-9219b6982323, session_id: d2b38da3-e438-4e27-b550-ef7f093a76bf, gen: 1 } is now active (read-write)
Sep 22 23:21:38.059 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.059 INFO d15c167b-c380-46a4-a953-864fc02495ed WaitActive WaitActive WaitActive
Sep 22 23:21:38.060 INFO [1] downstairs client at 127.0.0.1:40280 has UUID 84d77caf-cd99-479d-b329-65e270f8e6d5
Sep 22 23:21:38.060 INFO UpstairsConnection { upstairs_id: e4fe1d81-dc04-41e4-b745-9219b6982323, session_id: d2b38da3-e438-4e27-b550-ef7f093a76bf, gen: 1 } is now active (read-write)
Sep 22 23:21:38.060 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 84d77caf-cd99-479d-b329-65e270f8e6d5, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.060 INFO d15c167b-c380-46a4-a953-864fc02495ed WaitActive WaitActive WaitActive
Sep 22 23:21:38.060 INFO [2] downstairs client at 127.0.0.1:36377 has UUID 1104ac55-f7a8-4cef-b93e-5cab8f0588d7
Sep 22 23:21:38.060 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1104ac55-f7a8-4cef-b93e-5cab8f0588d7, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.060 INFO d15c167b-c380-46a4-a953-864fc02495ed WaitActive WaitActive WaitActive
Sep 22 23:21:38.060 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.060 DEBG [1] Read AckReady 1001, : downstairs
Sep 22 23:21:38.060 INFO [0] downstairs client at 127.0.0.1:48412 has UUID f00b2386-14a7-47f7-ad70-b49b4e8545ea
Sep 22 23:21:38.060 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f00b2386-14a7-47f7-ad70-b49b4e8545ea, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.060 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.060 INFO e4fe1d81-dc04-41e4-b745-9219b6982323 WaitActive WaitActive WaitActive
Sep 22 23:21:38.060 DEBG [0] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.060 INFO [1] downstairs client at 127.0.0.1:41943 has UUID c47df1e6-d581-419f-94ff-178a0e67a3ed
Sep 22 23:21:38.060 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c47df1e6-d581-419f-94ff-178a0e67a3ed, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.060 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.060 INFO e4fe1d81-dc04-41e4-b745-9219b6982323 WaitActive WaitActive WaitActive
Sep 22 23:21:38.060 INFO [2] downstairs client at 127.0.0.1:33214 has UUID d170e9eb-99a7-40aa-8e4b-74aea23bd418
Sep 22 23:21:38.060 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.060 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d170e9eb-99a7-40aa-8e4b-74aea23bd418, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.060 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.060 INFO e4fe1d81-dc04-41e4-b745-9219b6982323 WaitActive WaitActive WaitActive
Sep 22 23:21:38.060 DEBG up_ds_listen was notified
Sep 22 23:21:38.060 DEBG up_ds_listen process 1001
Sep 22 23:21:38.060 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.061 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.061 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.061 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.061 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.061 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.061 INFO [0] d15c167b-c380-46a4-a953-864fc02495ed (d8b1013b-fbba-4bb2-a01a-1a978b72bafa) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.061 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.061 WARN [0] new RM replaced this: None
Sep 22 23:21:38.061 INFO [0] Starts reconcile loop
Sep 22 23:21:38.061 INFO [1] d15c167b-c380-46a4-a953-864fc02495ed (d8b1013b-fbba-4bb2-a01a-1a978b72bafa) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.061 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.061 WARN [1] new RM replaced this: None
Sep 22 23:21:38.061 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.061 INFO [1] Starts reconcile loop
Sep 22 23:21:38.061 INFO [2] d15c167b-c380-46a4-a953-864fc02495ed (d8b1013b-fbba-4bb2-a01a-1a978b72bafa) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.061 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.061 WARN [2] new RM replaced this: None
Sep 22 23:21:38.061 INFO [2] Starts reconcile loop
Sep 22 23:21:38.061 INFO [0] 127.0.0.1:33968 task reports connection:true
Sep 22 23:21:38.061 INFO d15c167b-c380-46a4-a953-864fc02495ed WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.061 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.061 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.061 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.061 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.061 DEBG IO Write 1002 has deps []
Sep 22 23:21:38.061 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.061 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.061 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.061 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.061 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.061 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.061 INFO Max found gen is 1
Sep 22 23:21:38.061 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.061 DEBG up_ds_listen was notified
Sep 22 23:21:38.061 INFO Next flush: 1
Sep 22 23:21:38.061 DEBG [1] Read already AckReady 1005, : downstairs
Sep 22 23:21:38.061 DEBG up_ds_listen process 1002
Sep 22 23:21:38.061 INFO All extents match
Sep 22 23:21:38.061 INFO No downstairs repair required
Sep 22 23:21:38.061 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.061 INFO No initial repair work was required
Sep 22 23:21:38.061 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.061 INFO d15c167b-c380-46a4-a953-864fc02495ed is now active with session: d8b1013b-fbba-4bb2-a01a-1a978b72bafa
Sep 22 23:21:38.061 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.061 INFO d15c167b-c380-46a4-a953-864fc02495ed Set Active after no repair
Sep 22 23:21:38.061 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.061 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.061 INFO Set check for repair
Sep 22 23:21:38.062 INFO [1] 127.0.0.1:40280 task reports connection:true
Sep 22 23:21:38.062 INFO d15c167b-c380-46a4-a953-864fc02495ed Active Active Active
Sep 22 23:21:38.062 INFO Set check for repair
Sep 22 23:21:38.062 INFO [2] 127.0.0.1:36377 task reports connection:true
Sep 22 23:21:38.062 INFO d15c167b-c380-46a4-a953-864fc02495ed Active Active Active
Sep 22 23:21:38.062 INFO Set check for repair
Sep 22 23:21:38.062 INFO [0] received reconcile message
Sep 22 23:21:38.062 INFO [0] All repairs completed, exit
Sep 22 23:21:38.062 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.062 INFO [0] Starts cmd_loop
Sep 22 23:21:38.062 INFO [1] received reconcile message
Sep 22 23:21:38.062 INFO [1] All repairs completed, exit
Sep 22 23:21:38.062 INFO [1] Starts cmd_loop
Sep 22 23:21:38.062 INFO [2] received reconcile message
Sep 22 23:21:38.062 INFO [2] All repairs completed, exit
Sep 22 23:21:38.062 INFO [2] Starts cmd_loop
Sep 22 23:21:38.062 INFO [0] e4fe1d81-dc04-41e4-b745-9219b6982323 (d2b38da3-e438-4e27-b550-ef7f093a76bf) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.062 INFO [0] Transition from WaitActive to WaitQuorum
The guest has finished waiting for activation
Sep 22 23:21:38.062 WARN [0] new RM replaced this: None
Sep 22 23:21:38.062 INFO [0] Starts reconcile loop
Sep 22 23:21:38.062 INFO [1] e4fe1d81-dc04-41e4-b745-9219b6982323 (d2b38da3-e438-4e27-b550-ef7f093a76bf) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.062 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.062 WARN [1] new RM replaced this: None
Sep 22 23:21:38.062 INFO [1] Starts reconcile loop
Sep 22 23:21:38.062 DEBG IO Read  1000 has deps []
Sep 22 23:21:38.062 INFO [2] e4fe1d81-dc04-41e4-b745-9219b6982323 (d2b38da3-e438-4e27-b550-ef7f093a76bf) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.062 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.062 WARN [2] new RM replaced this: None
Sep 22 23:21:38.062 INFO [2] Starts reconcile loop
Sep 22 23:21:38.062 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.062 INFO [0] 127.0.0.1:48412 task reports connection:true
Sep 22 23:21:38.062 INFO e4fe1d81-dc04-41e4-b745-9219b6982323 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.062 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.062 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.062 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.062 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.063 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.063 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.063 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.063 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.063 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.063 INFO Max found gen is 1
Sep 22 23:21:38.063 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.063 INFO Next flush: 1
Sep 22 23:21:38.063 INFO All extents match
Sep 22 23:21:38.063 INFO No downstairs repair required
Sep 22 23:21:38.063 INFO No initial repair work was required
Sep 22 23:21:38.063 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.063 INFO e4fe1d81-dc04-41e4-b745-9219b6982323 is now active with session: d2b38da3-e438-4e27-b550-ef7f093a76bf
Sep 22 23:21:38.063 INFO e4fe1d81-dc04-41e4-b745-9219b6982323 Set Active after no repair
Sep 22 23:21:38.063 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.063 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.063 INFO Set check for repair
Sep 22 23:21:38.063 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.063 INFO [1] 127.0.0.1:41943 task reports connection:true
Sep 22 23:21:38.063 INFO e4fe1d81-dc04-41e4-b745-9219b6982323 Active Active Active
Sep 22 23:21:38.063 INFO Set check for repair
Sep 22 23:21:38.063 INFO [2] 127.0.0.1:33214 task reports connection:true
Sep 22 23:21:38.063 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.063 INFO e4fe1d81-dc04-41e4-b745-9219b6982323 Active Active Active
Sep 22 23:21:38.063 INFO Set check for repair
Sep 22 23:21:38.063 INFO [0] received reconcile message
Sep 22 23:21:38.063 INFO [0] All repairs completed, exit
Sep 22 23:21:38.063 INFO [0] Starts cmd_loop
Sep 22 23:21:38.063 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.063 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.063 INFO [1] received reconcile message
Sep 22 23:21:38.063 INFO [1] All repairs completed, exit
Sep 22 23:21:38.063 INFO [1] Starts cmd_loop
Sep 22 23:21:38.063 INFO [2] received reconcile message
Sep 22 23:21:38.063 INFO [2] All repairs completed, exit
Sep 22 23:21:38.063 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:38.063 INFO [2] Starts cmd_loop
Sep 22 23:21:38.063 DEBG [1] Read already AckReady 1000, : downstairs
The guest has finished waiting for activation
Sep 22 23:21:38.064 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:38.064 DEBG up_ds_listen was notified
Sep 22 23:21:38.064 DEBG up_ds_listen process 1000
Sep 22 23:21:38.064 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.064 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.064 DEBG IO Write 1002 has deps []
Sep 22 23:21:38.064 DEBG up_ds_listen was notified
Sep 22 23:21:38.064 DEBG up_ds_listen process 1002
Sep 22 23:21:38.064 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.064 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.064 DEBG [2] Read already AckReady 1005, : downstairs
Sep 22 23:21:38.064 DEBG up_ds_listen was notified
Sep 22 23:21:38.064 DEBG up_ds_listen process 1005
Sep 22 23:21:38.064 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:21:38.064 INFO Scrub check for df9234d2-5959-4534-87ed-4a79e641aa2a
Sep 22 23:21:38.064 INFO Scrub for df9234d2-5959-4534-87ed-4a79e641aa2a begins
Sep 22 23:21:38.064 INFO Scrub with total_size:7680 block_size:512
Sep 22 23:21:38.064 INFO Scrubs from block 0 to 15 in (256) 131072 size IOs pm:0
Sep 22 23:21:38.064 INFO Adjust block_count to 15 at offset 0
Sep 22 23:21:38.065 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.065 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:38.065 DEBG IO Read  1003 has deps []
Sep 22 23:21:38.065 DEBG up_ds_listen was notified
Sep 22 23:21:38.065 DEBG up_ds_listen process 1001
Sep 22 23:21:38.065 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.065 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.066 DEBG Read      :1003 deps:[] res:true
Sep 22 23:21:38.066 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.066 DEBG Read      :1003 deps:[] res:true
Sep 22 23:21:38.066 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.066 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.066 DEBG Read      :1003 deps:[] res:true
Sep 22 23:21:38.066 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.067 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.067 DEBG up_ds_listen was notified
Sep 22 23:21:38.067 DEBG up_ds_listen process 1000
Sep 22 23:21:38.067 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.067 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.067 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.067 DEBG IO Read  1002 has deps [JobId(1001)]
Sep 22 23:21:38.067 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:38.067 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.067 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:21:38.067 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:38.068 DEBG Read      :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:38.068 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.068 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.068 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:21:38.068 DEBG up_ds_listen was notified
Sep 22 23:21:38.068 DEBG up_ds_listen process 1003
Sep 22 23:21:38.068 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:38.068 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:21:38.069 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:21:38.069 DEBG up_ds_listen was notified
Sep 22 23:21:38.069 DEBG up_ds_listen process 1002
Sep 22 23:21:38.069 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.069 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.069 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.069 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.070 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.070 DEBG IO Write 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:21:38.071 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.071 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:21:38.072 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.073 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.074 DEBG Read      :1001 deps:[JobId(1000)] res:true
test test::integration_test_url ... ok
Sep 22 23:21:38.076 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.077 INFO Created new region file "/tmp/downstairs-xCnpISU1/region.json"
Sep 22 23:21:38.077 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:38.077 DEBG up_ds_listen was notified
test test::integration_test_volume_subvols_parent_scrub_sparse ... ok
Sep 22 23:21:38.077 DEBG up_ds_listen process 1003
Sep 22 23:21:38.077 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:38.077 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.077 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.077 INFO Created new region file "/tmp/downstairs-2ztTe6bf/region.json"
Sep 22 23:21:38.079 DEBG IO Write 1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:21:38.079 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.079 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.079 INFO Opened existing region file "/tmp/downstairs-xCnpISU1/region.json"
Sep 22 23:21:38.079 INFO Database read version 1
Sep 22 23:21:38.079 INFO Database write version 1
Sep 22 23:21:38.080 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.080 INFO Opened existing region file "/tmp/downstairs-2ztTe6bf/region.json"
Sep 22 23:21:38.080 INFO Database read version 1
Sep 22 23:21:38.080 INFO Database write version 1
Sep 22 23:21:38.081 INFO UUID: cdee1758-9349-418c-9b8d-629486998c09
Sep 22 23:21:38.081 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.081 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.081 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.081 INFO Using address: 127.0.0.1:48328, task: main
Sep 22 23:21:38.081 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.081 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33609, task: repair
Sep 22 23:21:38.081 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33609, task: repair
Sep 22 23:21:38.081 INFO listening, local_addr: 127.0.0.1:33609, task: repair
Sep 22 23:21:38.081 DEBG up_ds_listen was notified
Sep 22 23:21:38.081 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.081 DEBG up_ds_listen process 1003
Sep 22 23:21:38.081 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:38.081 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33609, task: repair
Sep 22 23:21:38.081 INFO Using repair address: 127.0.0.1:33609, task: main
Sep 22 23:21:38.081 DEBG up_ds_listen was notified
Sep 22 23:21:38.081 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.081 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.081 DEBG up_ds_listen process 1001
Sep 22 23:21:38.081 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.081 INFO Scrub at offset 15/15 sp:15
Sep 22 23:21:38.082 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.082 INFO Created new region file "/tmp/downstairs-oZLK1PnT/region.json"
Sep 22 23:21:38.082 INFO UUID: 01c46ab7-4671-4b4c-8a3d-a1a31f152199
Sep 22 23:21:38.082 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.082 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.082 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.082 INFO Using address: 127.0.0.1:58203, task: main
Sep 22 23:21:38.082 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.082 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.082 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38864, task: repair
Sep 22 23:21:38.082 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38864, task: repair
Sep 22 23:21:38.082 INFO listening, local_addr: 127.0.0.1:38864, task: repair
Sep 22 23:21:38.082 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38864, task: repair
Sep 22 23:21:38.082 INFO Using repair address: 127.0.0.1:38864, task: main
Sep 22 23:21:38.082 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.083 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.083 INFO Scrub df9234d2-5959-4534-87ed-4a79e641aa2a done in 0 seconds. Retries:0 scrub_size:7680 size:15 pause_milli:0
Sep 22 23:21:38.083 DEBG IO Flush 1004 has deps [JobId(1003), JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:21:38.083 INFO Created new region file "/tmp/downstairs-pi6CxBYH/region.json"
Sep 22 23:21:38.084 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:38.085 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.085 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.085 INFO Opened existing region file "/tmp/downstairs-oZLK1PnT/region.json"
Sep 22 23:21:38.085 INFO Database read version 1
Sep 22 23:21:38.085 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.085 INFO Database write version 1
Sep 22 23:21:38.086 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.086 DEBG up_ds_listen was notified
Sep 22 23:21:38.086 DEBG up_ds_listen process 1004
Sep 22 23:21:38.086 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:21:38.086 DEBG [rc] retire 1004 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:21:38.086 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.086 DEBG up_ds_listen was notified
Sep 22 23:21:38.086 DEBG up_ds_listen process 1002
Sep 22 23:21:38.086 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.086 DEBG IO Flush 1004 has deps [JobId(1003), JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:21:38.086 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.086 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
Sep 22 23:21:38.087 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.087 INFO Opened existing region file "/tmp/downstairs-pi6CxBYH/region.json"
Sep 22 23:21:38.087 INFO Database read version 1
Sep 22 23:21:38.087 INFO Database write version 1
Sep 22 23:21:38.087 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.087 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.088 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.088 INFO UUID: 5df431a9-e8cc-4535-a933-0ec6a3392f80
Sep 22 23:21:38.088 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.088 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.088 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.088 INFO Using address: 127.0.0.1:62933, task: main
Sep 22 23:21:38.088 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.088 INFO UUID: 08ca7a26-86b3-4310-8bb9-1c7dab6c4622
Sep 22 23:21:38.088 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.088 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43692, task: repair
Sep 22 23:21:38.088 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.088 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.088 INFO Using address: 127.0.0.1:44218, task: main
Sep 22 23:21:38.088 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43692, task: repair
Sep 22 23:21:38.089 INFO listening, local_addr: 127.0.0.1:43692, task: repair
Sep 22 23:21:38.089 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.089 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.089 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43692, task: repair
Sep 22 23:21:38.089 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.089 INFO Using repair address: 127.0.0.1:43692, task: main
Sep 22 23:21:38.089 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.089 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35997, task: repair
Sep 22 23:21:38.089 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35997, task: repair
Sep 22 23:21:38.089 INFO listening, local_addr: 127.0.0.1:35997, task: repair
Sep 22 23:21:38.089 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.089 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.089 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35997, task: repair
Sep 22 23:21:38.089 INFO Using repair address: 127.0.0.1:35997, task: main
Sep 22 23:21:38.089 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.089 INFO Created new region file "/tmp/downstairs-3f3WKQgh/region.json"
Sep 22 23:21:38.089 DEBG up_ds_listen was notified
Sep 22 23:21:38.089 DEBG up_ds_listen process 1004
Sep 22 23:21:38.089 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:21:38.089 DEBG [rc] retire 1004 clears [JobId(1000), JobId(1001), JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:21:38.089 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.089 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.089 INFO Created new region file "/tmp/downstairs-8RaWELXM/region.json"
Sep 22 23:21:38.090 DEBG IO Read  1005 has deps []
Sep 22 23:21:38.090 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:21:38.091 DEBG Read      :1005 deps:[] res:true
Sep 22 23:21:38.091 DEBG Read      :1005 deps:[] res:true
Sep 22 23:21:38.092 DEBG Read      :1005 deps:[] res:true
Sep 22 23:21:38.092 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.093 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.093 INFO Opened existing region file "/tmp/downstairs-3f3WKQgh/region.json"
Sep 22 23:21:38.093 INFO Database read version 1
Sep 22 23:21:38.093 INFO Database write version 1
Sep 22 23:21:38.093 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.093 INFO Opened existing region file "/tmp/downstairs-8RaWELXM/region.json"
Sep 22 23:21:38.093 INFO Database read version 1
Sep 22 23:21:38.093 INFO Database write version 1
Sep 22 23:21:38.094 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.094 DEBG up_ds_listen was notified
Sep 22 23:21:38.094 DEBG up_ds_listen process 1003
Sep 22 23:21:38.094 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:38.094 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.095 INFO UUID: ac9e9e82-be8d-4c20-a78c-e47f39a6b49e
Sep 22 23:21:38.096 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.096 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.096 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.096 INFO Using address: 127.0.0.1:65141, task: main
Sep 22 23:21:38.096 DEBG [0] Read AckReady 1005, : downstairs
Sep 22 23:21:38.096 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.096 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50256, task: repair
Sep 22 23:21:38.096 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50256, task: repair
Sep 22 23:21:38.096 INFO UUID: 6f2fbc1f-a9a2-41d4-9856-3b247141fc3f
Sep 22 23:21:38.096 INFO listening, local_addr: 127.0.0.1:50256, task: repair
Sep 22 23:21:38.096 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.096 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.096 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.096 INFO Using address: 127.0.0.1:52819, task: main
Sep 22 23:21:38.096 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50256, task: repair
Sep 22 23:21:38.096 INFO Using repair address: 127.0.0.1:50256, task: main
Sep 22 23:21:38.096 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.096 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.097 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56254, task: repair
Sep 22 23:21:38.097 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56254, task: repair
Sep 22 23:21:38.097 INFO listening, local_addr: 127.0.0.1:56254, task: repair
Sep 22 23:21:38.097 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56254, task: repair
Sep 22 23:21:38.097 INFO Using repair address: 127.0.0.1:56254, task: main
Sep 22 23:21:38.097 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.097 INFO Upstairs starts
Sep 22 23:21:38.097 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.097 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.097 INFO Crucible stats registered with UUID: 5b90282f-6bb7-4b5b-a1a9-64d086158b57
Sep 22 23:21:38.097 INFO Crucible 5b90282f-6bb7-4b5b-a1a9-64d086158b57 has session id: 9d7d57fb-4df7-4fab-a327-c17718952c14
Sep 22 23:21:38.097 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.097 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.097 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.097 INFO [0] connecting to 127.0.0.1:48328, looper: 0
Sep 22 23:21:38.097 INFO [1] connecting to 127.0.0.1:62933, looper: 1
Sep 22 23:21:38.097 INFO [2] connecting to 127.0.0.1:65141, looper: 2
Sep 22 23:21:38.098 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.098 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.098 INFO Flush timeout: 0.5
Sep 22 23:21:38.098 INFO Upstairs starts
Sep 22 23:21:38.098 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.098 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.098 INFO Crucible stats registered with UUID: 826051a1-df7d-43b5-adbe-a9bb32b9d0ea
Sep 22 23:21:38.098 INFO Crucible 826051a1-df7d-43b5-adbe-a9bb32b9d0ea has session id: 005ed4bb-0eb0-47f1-b9b2-0af5c1a94d59
test test::integration_test_volume_write_unwritten_1 ... okSep 22 23:21:38.098 INFO [0] 5b90282f-6bb7-4b5b-a1a9-64d086158b57 looper connected, looper: 0

Sep 22 23:21:38.098 INFO [0] Proc runs for 127.0.0.1:48328 in state New
Sep 22 23:21:38.098 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.098 INFO accepted connection from 127.0.0.1:53646, task: main
Sep 22 23:21:38.098 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.098 INFO accepted connection from 127.0.0.1:61364, task: main
Sep 22 23:21:38.098 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.098 INFO [1] 5b90282f-6bb7-4b5b-a1a9-64d086158b57 looper connected, looper: 1
Sep 22 23:21:38.098 INFO [1] Proc runs for 127.0.0.1:62933 in state New
Sep 22 23:21:38.098 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.098 INFO [2] 5b90282f-6bb7-4b5b-a1a9-64d086158b57 looper connected, looper: 2
Sep 22 23:21:38.098 INFO Created new region file "/tmp/downstairs-GpQSlTJR/region.json"
Sep 22 23:21:38.098 INFO [2] Proc runs for 127.0.0.1:65141 in state New
Sep 22 23:21:38.098 INFO [0] connecting to 127.0.0.1:58203, looper: 0
Sep 22 23:21:38.098 INFO accepted connection from 127.0.0.1:60140, task: main
Sep 22 23:21:38.098 INFO [1] connecting to 127.0.0.1:44218, looper: 1
Sep 22 23:21:38.098 DEBG [1] Read already AckReady 1005, : downstairs
Sep 22 23:21:38.098 INFO [2] connecting to 127.0.0.1:52819, looper: 2
Sep 22 23:21:38.098 INFO Connection request from 5b90282f-6bb7-4b5b-a1a9-64d086158b57 with version 4, task: proc
Sep 22 23:21:38.098 INFO upstairs UpstairsConnection { upstairs_id: 5b90282f-6bb7-4b5b-a1a9-64d086158b57, session_id: 2d00b74f-dde3-4064-ba30-0de199ff9cba, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.099 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.099 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.099 INFO Flush timeout: 0.5
Sep 22 23:21:38.099 INFO Connection request from 5b90282f-6bb7-4b5b-a1a9-64d086158b57 with version 4, task: proc
Sep 22 23:21:38.099 INFO upstairs UpstairsConnection { upstairs_id: 5b90282f-6bb7-4b5b-a1a9-64d086158b57, session_id: 2d00b74f-dde3-4064-ba30-0de199ff9cba, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.099 INFO Connection request from 5b90282f-6bb7-4b5b-a1a9-64d086158b57 with version 4, task: proc
Sep 22 23:21:38.099 INFO upstairs UpstairsConnection { upstairs_id: 5b90282f-6bb7-4b5b-a1a9-64d086158b57, session_id: 2d00b74f-dde3-4064-ba30-0de199ff9cba, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.099 INFO accepted connection from 127.0.0.1:46749, task: main
Sep 22 23:21:38.099 INFO accepted connection from 127.0.0.1:65098, task: main
Sep 22 23:21:38.099 INFO [0] 5b90282f-6bb7-4b5b-a1a9-64d086158b57 (2d00b74f-dde3-4064-ba30-0de199ff9cba) New New New ds_transition to WaitActive
Sep 22 23:21:38.099 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.099 INFO [1] 5b90282f-6bb7-4b5b-a1a9-64d086158b57 (2d00b74f-dde3-4064-ba30-0de199ff9cba) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.099 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.099 INFO [0] 826051a1-df7d-43b5-adbe-a9bb32b9d0ea looper connected, looper: 0
Sep 22 23:21:38.099 INFO [2] 5b90282f-6bb7-4b5b-a1a9-64d086158b57 (2d00b74f-dde3-4064-ba30-0de199ff9cba) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.099 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.099 INFO [0] Proc runs for 127.0.0.1:58203 in state New
Sep 22 23:21:38.099 INFO [1] 826051a1-df7d-43b5-adbe-a9bb32b9d0ea looper connected, looper: 1
The guest has requested activation
Sep 22 23:21:38.099 INFO 5b90282f-6bb7-4b5b-a1a9-64d086158b57 active request set
Sep 22 23:21:38.099 INFO [1] Proc runs for 127.0.0.1:44218 in state New
Sep 22 23:21:38.099 INFO [0] received activate with gen 1
Sep 22 23:21:38.099 INFO [2] 826051a1-df7d-43b5-adbe-a9bb32b9d0ea looper connected, looper: 2
Sep 22 23:21:38.099 INFO [0] client got ds_active_rx, promote! session 2d00b74f-dde3-4064-ba30-0de199ff9cba
Sep 22 23:21:38.099 INFO [2] Proc runs for 127.0.0.1:52819 in state New
Sep 22 23:21:38.099 INFO [1] received activate with gen 1
Sep 22 23:21:38.099 INFO [1] client got ds_active_rx, promote! session 2d00b74f-dde3-4064-ba30-0de199ff9cba
Sep 22 23:21:38.099 INFO [2] received activate with gen 1
Sep 22 23:21:38.099 INFO [2] client got ds_active_rx, promote! session 2d00b74f-dde3-4064-ba30-0de199ff9cba
Sep 22 23:21:38.099 INFO accepted connection from 127.0.0.1:46152, task: main
Sep 22 23:21:38.099 INFO UpstairsConnection { upstairs_id: 5b90282f-6bb7-4b5b-a1a9-64d086158b57, session_id: 2d00b74f-dde3-4064-ba30-0de199ff9cba, gen: 1 } is now active (read-write)
Sep 22 23:21:38.100 INFO UpstairsConnection { upstairs_id: 5b90282f-6bb7-4b5b-a1a9-64d086158b57, session_id: 2d00b74f-dde3-4064-ba30-0de199ff9cba, gen: 1 } is now active (read-write)
Sep 22 23:21:38.100 INFO UpstairsConnection { upstairs_id: 5b90282f-6bb7-4b5b-a1a9-64d086158b57, session_id: 2d00b74f-dde3-4064-ba30-0de199ff9cba, gen: 1 } is now active (read-write)
Sep 22 23:21:38.100 INFO Connection request from 826051a1-df7d-43b5-adbe-a9bb32b9d0ea with version 4, task: proc
Sep 22 23:21:38.100 INFO upstairs UpstairsConnection { upstairs_id: 826051a1-df7d-43b5-adbe-a9bb32b9d0ea, session_id: 77674847-2bf1-45c2-9d96-dadd8139fd25, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.100 INFO Connection request from 826051a1-df7d-43b5-adbe-a9bb32b9d0ea with version 4, task: proc
Sep 22 23:21:38.100 INFO upstairs UpstairsConnection { upstairs_id: 826051a1-df7d-43b5-adbe-a9bb32b9d0ea, session_id: 77674847-2bf1-45c2-9d96-dadd8139fd25, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.100 INFO Connection request from 826051a1-df7d-43b5-adbe-a9bb32b9d0ea with version 4, task: proc
Sep 22 23:21:38.100 INFO upstairs UpstairsConnection { upstairs_id: 826051a1-df7d-43b5-adbe-a9bb32b9d0ea, session_id: 77674847-2bf1-45c2-9d96-dadd8139fd25, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.100 INFO [0] downstairs client at 127.0.0.1:48328 has UUID cdee1758-9349-418c-9b8d-629486998c09
Sep 22 23:21:38.100 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: cdee1758-9349-418c-9b8d-629486998c09, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.100 INFO 5b90282f-6bb7-4b5b-a1a9-64d086158b57 WaitActive WaitActive WaitActive
Sep 22 23:21:38.100 INFO [1] downstairs client at 127.0.0.1:62933 has UUID 5df431a9-e8cc-4535-a933-0ec6a3392f80
Sep 22 23:21:38.100 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 5df431a9-e8cc-4535-a933-0ec6a3392f80, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.100 DEBG [2] Read already AckReady 1005, : downstairs
Sep 22 23:21:38.100 INFO 5b90282f-6bb7-4b5b-a1a9-64d086158b57 WaitActive WaitActive WaitActive
Sep 22 23:21:38.100 DEBG up_ds_listen was notified
Sep 22 23:21:38.100 DEBG up_ds_listen process 1005
Sep 22 23:21:38.100 INFO [0] 826051a1-df7d-43b5-adbe-a9bb32b9d0ea (77674847-2bf1-45c2-9d96-dadd8139fd25) New New New ds_transition to WaitActive
Sep 22 23:21:38.100 INFO [2] downstairs client at 127.0.0.1:65141 has UUID ac9e9e82-be8d-4c20-a78c-e47f39a6b49e
Sep 22 23:21:38.100 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:21:38.100 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ac9e9e82-be8d-4c20-a78c-e47f39a6b49e, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.100 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.100 INFO 5b90282f-6bb7-4b5b-a1a9-64d086158b57 WaitActive WaitActive WaitActive
Sep 22 23:21:38.100 INFO [1] 826051a1-df7d-43b5-adbe-a9bb32b9d0ea (77674847-2bf1-45c2-9d96-dadd8139fd25) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.100 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.100 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.100 INFO [2] 826051a1-df7d-43b5-adbe-a9bb32b9d0ea (77674847-2bf1-45c2-9d96-dadd8139fd25) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.100 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.101 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.101 INFO Downstairs has completed Negotiation, task: proc
The guest has requested activation
Sep 22 23:21:38.101 INFO 826051a1-df7d-43b5-adbe-a9bb32b9d0ea active request set
Sep 22 23:21:38.101 INFO [0] received activate with gen 1
Sep 22 23:21:38.101 DEBG IO Read  1005 has deps []
Sep 22 23:21:38.101 INFO [0] client got ds_active_rx, promote! session 77674847-2bf1-45c2-9d96-dadd8139fd25
Sep 22 23:21:38.101 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.101 INFO [1] received activate with gen 1
Sep 22 23:21:38.101 INFO [1] client got ds_active_rx, promote! session 77674847-2bf1-45c2-9d96-dadd8139fd25
Sep 22 23:21:38.101 INFO [2] received activate with gen 1
Sep 22 23:21:38.101 INFO [2] client got ds_active_rx, promote! session 77674847-2bf1-45c2-9d96-dadd8139fd25
Sep 22 23:21:38.101 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.101 INFO UpstairsConnection { upstairs_id: 826051a1-df7d-43b5-adbe-a9bb32b9d0ea, session_id: 77674847-2bf1-45c2-9d96-dadd8139fd25, gen: 1 } is now active (read-write)
Sep 22 23:21:38.101 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.101 INFO Opened existing region file "/tmp/downstairs-GpQSlTJR/region.json"
Sep 22 23:21:38.101 INFO Database read version 1
Sep 22 23:21:38.101 INFO Database write version 1
Sep 22 23:21:38.101 INFO UpstairsConnection { upstairs_id: 826051a1-df7d-43b5-adbe-a9bb32b9d0ea, session_id: 77674847-2bf1-45c2-9d96-dadd8139fd25, gen: 1 } is now active (read-write)
Sep 22 23:21:38.101 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.101 INFO UpstairsConnection { upstairs_id: 826051a1-df7d-43b5-adbe-a9bb32b9d0ea, session_id: 77674847-2bf1-45c2-9d96-dadd8139fd25, gen: 1 } is now active (read-write)
Sep 22 23:21:38.101 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.102 INFO [0] 5b90282f-6bb7-4b5b-a1a9-64d086158b57 (2d00b74f-dde3-4064-ba30-0de199ff9cba) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.102 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.102 WARN [0] new RM replaced this: None
Sep 22 23:21:38.102 INFO [0] Starts reconcile loop
Sep 22 23:21:38.102 INFO [1] 5b90282f-6bb7-4b5b-a1a9-64d086158b57 (2d00b74f-dde3-4064-ba30-0de199ff9cba) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.102 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.102 WARN [1] new RM replaced this: None
Sep 22 23:21:38.102 INFO [0] downstairs client at 127.0.0.1:58203 has UUID 01c46ab7-4671-4b4c-8a3d-a1a31f152199
Sep 22 23:21:38.102 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 01c46ab7-4671-4b4c-8a3d-a1a31f152199, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.102 INFO [1] Starts reconcile loop
Sep 22 23:21:38.102 INFO 826051a1-df7d-43b5-adbe-a9bb32b9d0ea WaitActive WaitActive WaitActive
Sep 22 23:21:38.102 INFO [2] 5b90282f-6bb7-4b5b-a1a9-64d086158b57 (2d00b74f-dde3-4064-ba30-0de199ff9cba) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.102 DEBG Read      :1005 deps:[] res:true
Sep 22 23:21:38.102 INFO [1] downstairs client at 127.0.0.1:44218 has UUID 08ca7a26-86b3-4310-8bb9-1c7dab6c4622
Sep 22 23:21:38.102 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.102 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 08ca7a26-86b3-4310-8bb9-1c7dab6c4622, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.102 WARN [2] new RM replaced this: None
Sep 22 23:21:38.102 INFO [2] Starts reconcile loop
Sep 22 23:21:38.102 INFO 826051a1-df7d-43b5-adbe-a9bb32b9d0ea WaitActive WaitActive WaitActive
Sep 22 23:21:38.102 INFO [2] downstairs client at 127.0.0.1:52819 has UUID 6f2fbc1f-a9a2-41d4-9856-3b247141fc3f
Sep 22 23:21:38.102 INFO [0] 127.0.0.1:48328 task reports connection:true
Sep 22 23:21:38.102 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6f2fbc1f-a9a2-41d4-9856-3b247141fc3f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.102 INFO 5b90282f-6bb7-4b5b-a1a9-64d086158b57 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.102 INFO 826051a1-df7d-43b5-adbe-a9bb32b9d0ea WaitActive WaitActive WaitActive
Sep 22 23:21:38.102 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.102 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.102 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.102 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.102 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.102 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.102 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.102 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.102 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.102 DEBG Read      :1005 deps:[] res:true
Sep 22 23:21:38.102 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.102 INFO Max found gen is 1
Sep 22 23:21:38.102 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.102 INFO Next flush: 1
Sep 22 23:21:38.102 INFO All extents match
Sep 22 23:21:38.102 INFO No downstairs repair required
Sep 22 23:21:38.102 INFO No initial repair work was required
Sep 22 23:21:38.102 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.102 INFO 5b90282f-6bb7-4b5b-a1a9-64d086158b57 is now active with session: 2d00b74f-dde3-4064-ba30-0de199ff9cba
Sep 22 23:21:38.102 INFO 5b90282f-6bb7-4b5b-a1a9-64d086158b57 Set Active after no repair
Sep 22 23:21:38.102 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.102 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.102 INFO Set check for repair
Sep 22 23:21:38.102 INFO [1] 127.0.0.1:62933 task reports connection:true
Sep 22 23:21:38.103 INFO 5b90282f-6bb7-4b5b-a1a9-64d086158b57 Active Active Active
Sep 22 23:21:38.103 INFO Set check for repair
Sep 22 23:21:38.103 DEBG Read      :1005 deps:[] res:true
Sep 22 23:21:38.103 INFO [2] 127.0.0.1:65141 task reports connection:true
Sep 22 23:21:38.103 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.103 INFO 5b90282f-6bb7-4b5b-a1a9-64d086158b57 Active Active Active
Sep 22 23:21:38.103 INFO Set check for repair
Sep 22 23:21:38.103 INFO [0] received reconcile message
Sep 22 23:21:38.103 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.103 INFO [0] All repairs completed, exit
Sep 22 23:21:38.103 INFO [0] Starts cmd_loop
Sep 22 23:21:38.103 INFO [1] received reconcile message
Sep 22 23:21:38.103 INFO [1] All repairs completed, exit
Sep 22 23:21:38.103 INFO [1] Starts cmd_loop
Sep 22 23:21:38.103 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.103 INFO [2] received reconcile message
Sep 22 23:21:38.103 INFO [2] All repairs completed, exit
Sep 22 23:21:38.103 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.103 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.103 INFO UUID: 1f4a8f3f-442f-484d-8ab0-408b22284518
Sep 22 23:21:38.103 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.103 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.103 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.103 INFO Using address: 127.0.0.1:49227, task: main
Sep 22 23:21:38.103 INFO [0] 826051a1-df7d-43b5-adbe-a9bb32b9d0ea (77674847-2bf1-45c2-9d96-dadd8139fd25) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.103 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.103 WARN [0] new RM replaced this: None
Sep 22 23:21:38.103 INFO [0] Starts reconcile loop
Sep 22 23:21:38.104 INFO [1] 826051a1-df7d-43b5-adbe-a9bb32b9d0ea (77674847-2bf1-45c2-9d96-dadd8139fd25) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.104 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.104 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.104 WARN [1] new RM replaced this: None
Sep 22 23:21:38.104 INFO [1] Starts reconcile loop
Sep 22 23:21:38.104 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61825, task: repair
Sep 22 23:21:38.104 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61825, task: repair
Sep 22 23:21:38.104 INFO [2] 826051a1-df7d-43b5-adbe-a9bb32b9d0ea (77674847-2bf1-45c2-9d96-dadd8139fd25) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.104 INFO listening, local_addr: 127.0.0.1:61825, task: repair
Sep 22 23:21:38.104 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.104 WARN [2] new RM replaced this: None
Sep 22 23:21:38.104 INFO [2] Starts reconcile loop
Sep 22 23:21:38.104 INFO [0] 127.0.0.1:58203 task reports connection:true
Sep 22 23:21:38.104 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61825, task: repair
Sep 22 23:21:38.104 INFO 826051a1-df7d-43b5-adbe-a9bb32b9d0ea WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.104 INFO Using repair address: 127.0.0.1:61825, task: main
Sep 22 23:21:38.104 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.104 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.104 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.104 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.104 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.104 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.104 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.104 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.104 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.104 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.104 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.104 INFO Max found gen is 1
Sep 22 23:21:38.104 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.104 INFO Next flush: 1
Sep 22 23:21:38.104 INFO All extents match
Sep 22 23:21:38.104 INFO Created new region file "/tmp/downstairs-wG6GScam/region.json"
Sep 22 23:21:38.104 INFO No downstairs repair required
Sep 22 23:21:38.104 INFO No initial repair work was required
Sep 22 23:21:38.104 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.104 INFO 826051a1-df7d-43b5-adbe-a9bb32b9d0ea is now active with session: 77674847-2bf1-45c2-9d96-dadd8139fd25
Sep 22 23:21:38.104 INFO 826051a1-df7d-43b5-adbe-a9bb32b9d0ea Set Active after no repair
Sep 22 23:21:38.104 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.104 INFO Set check for repair
Sep 22 23:21:38.104 INFO [1] 127.0.0.1:44218 task reports connection:true
Sep 22 23:21:38.104 INFO 826051a1-df7d-43b5-adbe-a9bb32b9d0ea Active Active Active
Sep 22 23:21:38.104 INFO Set check for repair
Sep 22 23:21:38.104 INFO [2] 127.0.0.1:52819 task reports connection:true
Sep 22 23:21:38.104 INFO 826051a1-df7d-43b5-adbe-a9bb32b9d0ea Active Active Active
Sep 22 23:21:38.104 INFO Set check for repair
Sep 22 23:21:38.105 INFO [0] received reconcile message
Sep 22 23:21:38.105 INFO [0] All repairs completed, exit
Sep 22 23:21:38.105 INFO [0] Starts cmd_loop
Sep 22 23:21:38.105 DEBG [1] Read AckReady 1005, : downstairs
Sep 22 23:21:38.105 INFO [1] received reconcile message
Sep 22 23:21:38.105 INFO [1] All repairs completed, exit
Sep 22 23:21:38.105 INFO [1] Starts cmd_loop
Sep 22 23:21:38.105 INFO [2] received reconcile message
Sep 22 23:21:38.105 INFO [2] All repairs completed, exit
Sep 22 23:21:38.105 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.105 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.106 DEBG up_ds_listen was notified
Sep 22 23:21:38.106 DEBG up_ds_listen process 1000
Sep 22 23:21:38.106 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.106 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.106 DEBG [0] Read already AckReady 1005, : downstairs
Sep 22 23:21:38.106 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.107 DEBG [2] Read already AckReady 1005, : downstairs
Sep 22 23:21:38.107 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.107 DEBG up_ds_listen was notified
Sep 22 23:21:38.107 DEBG up_ds_listen process 1005
Sep 22 23:21:38.107 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:21:38.107 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.107 INFO Opened existing region file "/tmp/downstairs-wG6GScam/region.json"
Sep 22 23:21:38.107 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.107 INFO Database read version 1
Sep 22 23:21:38.107 INFO Database write version 1
Sep 22 23:21:38.107 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.108 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.110 INFO UUID: b8e5d869-788a-45ac-ae38-e8f5cf16b888
Sep 22 23:21:38.110 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.110 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.110 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.110 INFO Using address: 127.0.0.1:44178, task: main
Sep 22 23:21:38.110 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.110 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48399, task: repair
Sep 22 23:21:38.110 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48399, task: repair
Sep 22 23:21:38.110 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:38.110 INFO listening, local_addr: 127.0.0.1:48399, task: repair
Sep 22 23:21:38.111 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48399, task: repair
Sep 22 23:21:38.111 INFO Using repair address: 127.0.0.1:48399, task: main
Sep 22 23:21:38.111 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.111 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.111 INFO Created new region file "/tmp/downstairs-VNELpEv2/region.json"
Sep 22 23:21:38.112 DEBG up_ds_listen was notified
Sep 22 23:21:38.112 DEBG up_ds_listen process 1000
Sep 22 23:21:38.112 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.112 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.112 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:21:38.113 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.113 DEBG Read      :1001 deps:[JobId(1000)] res:true
test test::integration_test_volume_subvols_parent_scrub_sparse_2 ... ok
Sep 22 23:21:38.114 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.114 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.114 INFO Created new region file "/tmp/downstairs-2f4SSbKQ/region.json"
Sep 22 23:21:38.115 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.115 INFO Opened existing region file "/tmp/downstairs-VNELpEv2/region.json"
Sep 22 23:21:38.115 INFO Database read version 1
Sep 22 23:21:38.115 INFO Database write version 1
Sep 22 23:21:38.116 INFO UUID: 8025f645-3aa5-482f-9a7b-5b38782d7449
Sep 22 23:21:38.116 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.116 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.116 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.116 INFO Using address: 127.0.0.1:53578, task: main
Sep 22 23:21:38.116 DEBG up_ds_listen was notified
Sep 22 23:21:38.116 DEBG up_ds_listen process 1001
Sep 22 23:21:38.116 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.116 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.116 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.116 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:65488, task: repair
Sep 22 23:21:38.116 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:65488, task: repair
Sep 22 23:21:38.116 INFO listening, local_addr: 127.0.0.1:65488, task: repair
Sep 22 23:21:38.117 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:65488, task: repair
Sep 22 23:21:38.117 INFO Using repair address: 127.0.0.1:65488, task: main
Sep 22 23:21:38.117 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.117 DEBG IO Read  1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:38.117 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.117 INFO Created new region file "/tmp/downstairs-H08UzdlI/region.json"
Sep 22 23:21:38.117 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.117 INFO Opened existing region file "/tmp/downstairs-2f4SSbKQ/region.json"
Sep 22 23:21:38.117 INFO Database read version 1
Sep 22 23:21:38.117 INFO Database write version 1
Sep 22 23:21:38.117 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:38.117 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.118 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.118 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.119 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.119 INFO UUID: 71f89d71-fca2-4667-8b12-c125e7772d19
Sep 22 23:21:38.119 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.119 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.119 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.119 INFO Using address: 127.0.0.1:51219, task: main
Sep 22 23:21:38.119 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.119 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57315, task: repair
Sep 22 23:21:38.119 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57315, task: repair
Sep 22 23:21:38.119 INFO listening, local_addr: 127.0.0.1:57315, task: repair
Sep 22 23:21:38.120 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57315, task: repair
Sep 22 23:21:38.120 INFO Using repair address: 127.0.0.1:57315, task: main
Sep 22 23:21:38.120 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.120 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.120 INFO Created new region file "/tmp/downstairs-mVZN7GWg/region.json"
Sep 22 23:21:38.121 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.121 DEBG up_ds_listen was notified
Sep 22 23:21:38.121 DEBG up_ds_listen process 1001
Sep 22 23:21:38.121 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.121 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.121 INFO Opened existing region file "/tmp/downstairs-H08UzdlI/region.json"
Sep 22 23:21:38.121 INFO Database read version 1
Sep 22 23:21:38.121 INFO Database write version 1
Sep 22 23:21:38.121 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.122 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:21:38.123 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:38.124 INFO UUID: 481d7429-1224-42e4-a2f2-287bd0cfeaaf
Sep 22 23:21:38.124 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.124 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.124 INFO Opened existing region file "/tmp/downstairs-mVZN7GWg/region.json"
Sep 22 23:21:38.124 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.124 INFO Database read version 1
Sep 22 23:21:38.124 INFO Database write version 1
Sep 22 23:21:38.124 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.124 INFO Using address: 127.0.0.1:58376, task: main
Sep 22 23:21:38.124 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.124 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52690, task: repair
Sep 22 23:21:38.124 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52690, task: repair
Sep 22 23:21:38.124 INFO listening, local_addr: 127.0.0.1:52690, task: repair
Sep 22 23:21:38.125 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52690, task: repair
Sep 22 23:21:38.125 INFO Using repair address: 127.0.0.1:52690, task: main
Sep 22 23:21:38.125 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.125 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:21:38.125 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.125 INFO Created new region file "/tmp/downstairs-L6iP4aDB/region.json"
Sep 22 23:21:38.125 DEBG up_ds_listen was notified
Sep 22 23:21:38.125 DEBG up_ds_listen process 1002
Sep 22 23:21:38.125 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.125 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.126 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
Sep 22 23:21:38.126 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.126 INFO UUID: 7c9decb7-dee6-4a23-a2b7-7860a60db1d6
Sep 22 23:21:38.126 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.127 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.127 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.127 INFO Using address: 127.0.0.1:44324, task: main
Sep 22 23:21:38.127 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.127 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.127 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39540, task: repair
Sep 22 23:21:38.127 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39540, task: repair
Sep 22 23:21:38.127 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.127 INFO listening, local_addr: 127.0.0.1:39540, task: repair
Sep 22 23:21:38.127 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39540, task: repair
Sep 22 23:21:38.127 INFO Using repair address: 127.0.0.1:39540, task: main
Sep 22 23:21:38.127 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.127 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:21:38.127 DEBG up_ds_listen was notified
Sep 22 23:21:38.127 DEBG up_ds_listen process 1002
Sep 22 23:21:38.127 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.128 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.128 INFO Created new region file "/tmp/downstairs-a3HQe4PA/region.json"
Sep 22 23:21:38.128 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.130 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:21:38.130 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.130 INFO Opened existing region file "/tmp/downstairs-L6iP4aDB/region.json"
Sep 22 23:21:38.130 INFO Database read version 1
Sep 22 23:21:38.130 INFO Database write version 1
Sep 22 23:21:38.132 DEBG [1] Read already AckReady 1003, : downstairs
test test::integration_test_volume_write_unwritten_sparse ... ok
Sep 22 23:21:38.132 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.132 INFO Opened existing region file "/tmp/downstairs-a3HQe4PA/region.json"
Sep 22 23:21:38.132 INFO Database read version 1
Sep 22 23:21:38.132 INFO UUID: f6750c55-7697-440f-81c9-8c2e90dcc4ce
Sep 22 23:21:38.132 INFO Database write version 1
Sep 22 23:21:38.132 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.132 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.133 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.133 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.133 INFO Using address: 127.0.0.1:47267, task: main
Sep 22 23:21:38.133 INFO Created new region file "/tmp/downstairs-cjWQt5F2/region.json"
Sep 22 23:21:38.133 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.133 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39855, task: repair
Sep 22 23:21:38.133 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39855, task: repair
Sep 22 23:21:38.133 INFO listening, local_addr: 127.0.0.1:39855, task: repair
Sep 22 23:21:38.133 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39855, task: repair
Sep 22 23:21:38.133 INFO Using repair address: 127.0.0.1:39855, task: main
Sep 22 23:21:38.133 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.133 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.133 DEBG up_ds_listen was notified
Sep 22 23:21:38.133 DEBG up_ds_listen process 1003
Sep 22 23:21:38.133 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:38.134 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.134 INFO Created new region file "/tmp/downstairs-qbyafYjG/region.json"
Sep 22 23:21:38.134 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.134 INFO UUID: 6e4f126c-20d7-47c5-9d94-936338c4e38d
Sep 22 23:21:38.135 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.135 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.135 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.135 INFO Using address: 127.0.0.1:52939, task: main
Sep 22 23:21:38.135 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.135 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53370, task: repair
Sep 22 23:21:38.135 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53370, task: repair
Sep 22 23:21:38.135 INFO listening, local_addr: 127.0.0.1:53370, task: repair
Sep 22 23:21:38.135 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53370, task: repair
Sep 22 23:21:38.136 INFO Using repair address: 127.0.0.1:53370, task: main
Sep 22 23:21:38.136 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.136 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.136 INFO Created new region file "/tmp/downstairs-Rn8xeVoQ/region.json"
Sep 22 23:21:38.137 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.137 INFO Opened existing region file "/tmp/downstairs-cjWQt5F2/region.json"
Sep 22 23:21:38.137 INFO Database read version 1
Sep 22 23:21:38.137 INFO Database write version 1
test test::integration_test_volume_write_unwritten_2 ... ok
Sep 22 23:21:38.139 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.139 INFO Created new region file "/tmp/downstairs-xyUXgpCG/region.json"
Sep 22 23:21:38.139 INFO UUID: 6a3090e7-a0f2-4a80-947f-1db172a31a5f
Sep 22 23:21:38.139 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.139 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.139 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.139 INFO Using address: 127.0.0.1:62388, task: main
Sep 22 23:21:38.140 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.140 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33943, task: repair
Sep 22 23:21:38.140 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33943, task: repair
Sep 22 23:21:38.140 INFO listening, local_addr: 127.0.0.1:33943, task: repair
Sep 22 23:21:38.140 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33943, task: repair
Sep 22 23:21:38.140 INFO Using repair address: 127.0.0.1:33943, task: main
Sep 22 23:21:38.140 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.140 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.140 INFO Opened existing region file "/tmp/downstairs-qbyafYjG/region.json"
Sep 22 23:21:38.140 INFO Database read version 1
Sep 22 23:21:38.140 INFO Database write version 1
Sep 22 23:21:38.141 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.141 INFO Created new region file "/tmp/downstairs-NfMGr4cT/region.json"
Sep 22 23:21:38.142 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.142 INFO Opened existing region file "/tmp/downstairs-Rn8xeVoQ/region.json"
Sep 22 23:21:38.142 INFO Database read version 1
Sep 22 23:21:38.142 INFO Database write version 1
Sep 22 23:21:38.143 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.143 INFO Opened existing region file "/tmp/downstairs-xyUXgpCG/region.json"
Sep 22 23:21:38.143 INFO Database read version 1
Sep 22 23:21:38.143 INFO Database write version 1
Sep 22 23:21:38.145 INFO UUID: 4bd08688-0a92-4b76-9dcc-9afb5f6f7cc8
Sep 22 23:21:38.145 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.145 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.145 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.145 INFO Using address: 127.0.0.1:54868, task: main
Sep 22 23:21:38.146 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.146 INFO UUID: 0d1968cb-b30f-4a44-a680-9f3b52cfb45a
Sep 22 23:21:38.146 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.146 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60848, task: repair
Sep 22 23:21:38.146 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60848, task: repair
Sep 22 23:21:38.146 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.146 INFO listening, local_addr: 127.0.0.1:60848, task: repair
Sep 22 23:21:38.146 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.146 INFO Using address: 127.0.0.1:48752, task: main
Sep 22 23:21:38.146 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60848, task: repair
Sep 22 23:21:38.146 INFO UUID: 07ed70f5-1d11-4ea6-9c11-5ff16ec9fcb3
Sep 22 23:21:38.146 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.146 INFO Using repair address: 127.0.0.1:60848, task: main
Sep 22 23:21:38.146 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.146 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.146 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.146 INFO Using address: 127.0.0.1:39369, task: main
Sep 22 23:21:38.146 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.146 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56982, task: repair
Sep 22 23:21:38.147 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56982, task: repair
Sep 22 23:21:38.147 INFO listening, local_addr: 127.0.0.1:56982, task: repair
Sep 22 23:21:38.147 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56982, task: repair
Sep 22 23:21:38.147 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.147 INFO Using repair address: 127.0.0.1:56982, task: main
Sep 22 23:21:38.147 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.147 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.147 INFO Opened existing region file "/tmp/downstairs-NfMGr4cT/region.json"
Sep 22 23:21:38.147 INFO Database read version 1
Sep 22 23:21:38.147 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38332, task: repair
Sep 22 23:21:38.147 INFO Database write version 1
Sep 22 23:21:38.147 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38332, task: repair
Sep 22 23:21:38.147 INFO listening, local_addr: 127.0.0.1:38332, task: repair
Sep 22 23:21:38.147 INFO Upstairs starts
Sep 22 23:21:38.147 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.147 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.147 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.147 INFO Crucible stats registered with UUID: 5e3310b1-e4cc-48c6-92c2-40a58c307dae
Sep 22 23:21:38.147 INFO Created new region file "/tmp/downstairs-dJ35WBCH/region.json"
Sep 22 23:21:38.147 INFO Crucible 5e3310b1-e4cc-48c6-92c2-40a58c307dae has session id: d0d5375e-5500-463d-ad8f-46e4f7ad8e6d
Sep 22 23:21:38.147 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38332, task: repair
Sep 22 23:21:38.147 INFO Using repair address: 127.0.0.1:38332, task: main
Sep 22 23:21:38.147 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.147 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.147 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.147 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.147 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.147 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.147 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.148 INFO [0] connecting to 127.0.0.1:49227, looper: 0
Sep 22 23:21:38.148 INFO [1] connecting to 127.0.0.1:44178, looper: 1
Sep 22 23:21:38.148 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.148 INFO [2] connecting to 127.0.0.1:53578, looper: 2
Sep 22 23:21:38.148 INFO Created new region file "/tmp/downstairs-W0JvBvQ4/region.json"
Sep 22 23:21:38.148 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.148 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.148 INFO Flush timeout: 0.5
Sep 22 23:21:38.148 INFO accepted connection from 127.0.0.1:52966, task: main
Sep 22 23:21:38.148 INFO accepted connection from 127.0.0.1:45823, task: main
Sep 22 23:21:38.148 INFO accepted connection from 127.0.0.1:33967, task: main
Sep 22 23:21:38.148 INFO [0] 5e3310b1-e4cc-48c6-92c2-40a58c307dae looper connected, looper: 0
Sep 22 23:21:38.148 INFO [0] Proc runs for 127.0.0.1:49227 in state New
Sep 22 23:21:38.148 INFO [1] 5e3310b1-e4cc-48c6-92c2-40a58c307dae looper connected, looper: 1
Sep 22 23:21:38.149 INFO [1] Proc runs for 127.0.0.1:44178 in state New
Sep 22 23:21:38.149 INFO [2] 5e3310b1-e4cc-48c6-92c2-40a58c307dae looper connected, looper: 2
Sep 22 23:21:38.149 INFO [2] Proc runs for 127.0.0.1:53578 in state New
Sep 22 23:21:38.149 INFO Upstairs starts
Sep 22 23:21:38.149 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.149 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.149 INFO Crucible stats registered with UUID: e89b9f37-20f1-48ba-8434-b618ba5c23aa
Sep 22 23:21:38.149 INFO Crucible e89b9f37-20f1-48ba-8434-b618ba5c23aa has session id: f7efcb29-a668-43c1-b069-9cfda4629be8
Sep 22 23:21:38.149 INFO UUID: ab814286-bbb3-4f61-a468-0a3953430633
Sep 22 23:21:38.149 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.149 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.149 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.149 INFO Using address: 127.0.0.1:43611, task: main
Sep 22 23:21:38.149 INFO Connection request from 5e3310b1-e4cc-48c6-92c2-40a58c307dae with version 4, task: proc
Sep 22 23:21:38.149 INFO upstairs UpstairsConnection { upstairs_id: 5e3310b1-e4cc-48c6-92c2-40a58c307dae, session_id: 2b11eab5-bc61-44bd-8216-a5695656c94c, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.149 INFO Connection request from 5e3310b1-e4cc-48c6-92c2-40a58c307dae with version 4, task: proc
Sep 22 23:21:38.149 INFO upstairs UpstairsConnection { upstairs_id: 5e3310b1-e4cc-48c6-92c2-40a58c307dae, session_id: 2b11eab5-bc61-44bd-8216-a5695656c94c, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.150 INFO Connection request from 5e3310b1-e4cc-48c6-92c2-40a58c307dae with version 4, task: proc
Sep 22 23:21:38.150 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.150 INFO upstairs UpstairsConnection { upstairs_id: 5e3310b1-e4cc-48c6-92c2-40a58c307dae, session_id: 2b11eab5-bc61-44bd-8216-a5695656c94c, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.150 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60439, task: repair
Sep 22 23:21:38.150 INFO [0] connecting to 127.0.0.1:58376, looper: 0
Sep 22 23:21:38.150 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60439, task: repair
Sep 22 23:21:38.150 INFO listening, local_addr: 127.0.0.1:60439, task: repair
Sep 22 23:21:38.150 INFO [1] connecting to 127.0.0.1:47267, looper: 1
Sep 22 23:21:38.150 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60439, task: repair
Sep 22 23:21:38.150 INFO Using repair address: 127.0.0.1:60439, task: main
Sep 22 23:21:38.150 INFO [2] connecting to 127.0.0.1:54868, looper: 2
Sep 22 23:21:38.150 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.150 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.150 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.150 INFO Flush timeout: 0.5
Sep 22 23:21:38.150 INFO accepted connection from 127.0.0.1:32910, task: main
Sep 22 23:21:38.150 INFO accepted connection from 127.0.0.1:42257, task: main
Sep 22 23:21:38.150 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.150 INFO [0] 5e3310b1-e4cc-48c6-92c2-40a58c307dae (2b11eab5-bc61-44bd-8216-a5695656c94c) New New New ds_transition to WaitActive
Sep 22 23:21:38.151 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.151 INFO Created new region file "/tmp/downstairs-IDeF0yuc/region.json"
Sep 22 23:21:38.151 INFO accepted connection from 127.0.0.1:42120, task: main
Sep 22 23:21:38.151 INFO [1] 5e3310b1-e4cc-48c6-92c2-40a58c307dae (2b11eab5-bc61-44bd-8216-a5695656c94c) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.151 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.151 INFO [2] 5e3310b1-e4cc-48c6-92c2-40a58c307dae (2b11eab5-bc61-44bd-8216-a5695656c94c) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.151 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.151 INFO [0] e89b9f37-20f1-48ba-8434-b618ba5c23aa looper connected, looper: 0
Sep 22 23:21:38.151 INFO [0] Proc runs for 127.0.0.1:58376 in state New
Sep 22 23:21:38.151 INFO [1] e89b9f37-20f1-48ba-8434-b618ba5c23aa looper connected, looper: 1
Sep 22 23:21:38.151 INFO [1] Proc runs for 127.0.0.1:47267 in state New
Sep 22 23:21:38.151 INFO [2] e89b9f37-20f1-48ba-8434-b618ba5c23aa looper connected, looper: 2
Sep 22 23:21:38.151 INFO [2] Proc runs for 127.0.0.1:54868 in state New
Sep 22 23:21:38.152 INFO Connection request from e89b9f37-20f1-48ba-8434-b618ba5c23aa with version 4, task: proc
Sep 22 23:21:38.152 INFO upstairs UpstairsConnection { upstairs_id: e89b9f37-20f1-48ba-8434-b618ba5c23aa, session_id: 902609d5-e2d5-4264-985b-fc9832ca6874, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.152 INFO Connection request from e89b9f37-20f1-48ba-8434-b618ba5c23aa with version 4, task: proc
Sep 22 23:21:38.152 INFO upstairs UpstairsConnection { upstairs_id: e89b9f37-20f1-48ba-8434-b618ba5c23aa, session_id: 902609d5-e2d5-4264-985b-fc9832ca6874, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.152 INFO Connection request from e89b9f37-20f1-48ba-8434-b618ba5c23aa with version 4, task: proc
Sep 22 23:21:38.152 INFO upstairs UpstairsConnection { upstairs_id: e89b9f37-20f1-48ba-8434-b618ba5c23aa, session_id: 902609d5-e2d5-4264-985b-fc9832ca6874, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.152 INFO [0] e89b9f37-20f1-48ba-8434-b618ba5c23aa (902609d5-e2d5-4264-985b-fc9832ca6874) New New New ds_transition to WaitActive
Sep 22 23:21:38.152 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.152 INFO [1] e89b9f37-20f1-48ba-8434-b618ba5c23aa (902609d5-e2d5-4264-985b-fc9832ca6874) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.152 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.152 INFO [2] e89b9f37-20f1-48ba-8434-b618ba5c23aa (902609d5-e2d5-4264-985b-fc9832ca6874) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.152 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.152 INFO current number of open files limit 65536 is already the maximum
The guest has requested activation
Sep 22 23:21:38.153 INFO Opened existing region file "/tmp/downstairs-W0JvBvQ4/region.json"
Sep 22 23:21:38.153 INFO Database read version 1
Sep 22 23:21:38.153 INFO Database write version 1
Sep 22 23:21:38.153 INFO 5e3310b1-e4cc-48c6-92c2-40a58c307dae active request set
Sep 22 23:21:38.153 INFO [0] received activate with gen 1
Sep 22 23:21:38.153 INFO [0] client got ds_active_rx, promote! session 2b11eab5-bc61-44bd-8216-a5695656c94c
Sep 22 23:21:38.153 INFO [1] received activate with gen 1
Sep 22 23:21:38.153 INFO [1] client got ds_active_rx, promote! session 2b11eab5-bc61-44bd-8216-a5695656c94c
Sep 22 23:21:38.153 INFO [2] received activate with gen 1
Sep 22 23:21:38.153 INFO [2] client got ds_active_rx, promote! session 2b11eab5-bc61-44bd-8216-a5695656c94c
Sep 22 23:21:38.153 INFO UpstairsConnection { upstairs_id: 5e3310b1-e4cc-48c6-92c2-40a58c307dae, session_id: 2b11eab5-bc61-44bd-8216-a5695656c94c, gen: 1 } is now active (read-write)
Sep 22 23:21:38.153 INFO UpstairsConnection { upstairs_id: 5e3310b1-e4cc-48c6-92c2-40a58c307dae, session_id: 2b11eab5-bc61-44bd-8216-a5695656c94c, gen: 1 } is now active (read-write)
Sep 22 23:21:38.153 INFO UpstairsConnection { upstairs_id: 5e3310b1-e4cc-48c6-92c2-40a58c307dae, session_id: 2b11eab5-bc61-44bd-8216-a5695656c94c, gen: 1 } is now active (read-write)
Sep 22 23:21:38.153 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.153 INFO Opened existing region file "/tmp/downstairs-dJ35WBCH/region.json"
Sep 22 23:21:38.153 INFO Database read version 1
Sep 22 23:21:38.153 INFO Database write version 1
Sep 22 23:21:38.154 INFO [0] downstairs client at 127.0.0.1:49227 has UUID 1f4a8f3f-442f-484d-8ab0-408b22284518
Sep 22 23:21:38.154 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1f4a8f3f-442f-484d-8ab0-408b22284518, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.154 INFO 5e3310b1-e4cc-48c6-92c2-40a58c307dae WaitActive WaitActive WaitActive
Sep 22 23:21:38.154 INFO [1] downstairs client at 127.0.0.1:44178 has UUID b8e5d869-788a-45ac-ae38-e8f5cf16b888
Sep 22 23:21:38.154 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: b8e5d869-788a-45ac-ae38-e8f5cf16b888, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.154 INFO 5e3310b1-e4cc-48c6-92c2-40a58c307dae WaitActive WaitActive WaitActive
Sep 22 23:21:38.154 INFO [2] downstairs client at 127.0.0.1:53578 has UUID 8025f645-3aa5-482f-9a7b-5b38782d7449
Sep 22 23:21:38.154 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 8025f645-3aa5-482f-9a7b-5b38782d7449, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.154 INFO 5e3310b1-e4cc-48c6-92c2-40a58c307dae WaitActive WaitActive WaitActive
Sep 22 23:21:38.154 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.155 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.155 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.155 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.155 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.155 INFO UUID: fbc1ff7a-5e2f-4a0a-a554-498b43e005a3
Sep 22 23:21:38.155 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.155 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.155 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.155 INFO Using address: 127.0.0.1:55268, task: main
Sep 22 23:21:38.155 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.155 INFO [0] 5e3310b1-e4cc-48c6-92c2-40a58c307dae (2b11eab5-bc61-44bd-8216-a5695656c94c) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.155 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.155 WARN [0] new RM replaced this: None
Sep 22 23:21:38.155 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.155 INFO [0] Starts reconcile loop
Sep 22 23:21:38.156 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50712, task: repair
Sep 22 23:21:38.156 INFO [1] 5e3310b1-e4cc-48c6-92c2-40a58c307dae (2b11eab5-bc61-44bd-8216-a5695656c94c) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.156 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50712, task: repair
Sep 22 23:21:38.156 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.156 WARN [1] new RM replaced this: None
Sep 22 23:21:38.156 INFO [1] Starts reconcile loop
Sep 22 23:21:38.156 INFO listening, local_addr: 127.0.0.1:50712, task: repair
Sep 22 23:21:38.156 INFO [2] 5e3310b1-e4cc-48c6-92c2-40a58c307dae (2b11eab5-bc61-44bd-8216-a5695656c94c) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.156 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.156 WARN [2] new RM replaced this: None
Sep 22 23:21:38.156 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50712, task: repair
Sep 22 23:21:38.156 INFO [2] Starts reconcile loop
Sep 22 23:21:38.156 INFO Using repair address: 127.0.0.1:50712, task: main
Sep 22 23:21:38.156 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.156 INFO [0] 127.0.0.1:49227 task reports connection:true
Sep 22 23:21:38.156 INFO 5e3310b1-e4cc-48c6-92c2-40a58c307dae WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.156 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.156 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.156 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.156 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.156 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.156 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.156 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.156 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.156 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.156 INFO Max found gen is 1
Sep 22 23:21:38.156 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.156 INFO Next flush: 1
Sep 22 23:21:38.156 INFO All extents match
Sep 22 23:21:38.156 INFO No downstairs repair required
Sep 22 23:21:38.156 INFO No initial repair work was required
Sep 22 23:21:38.156 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.156 INFO 5e3310b1-e4cc-48c6-92c2-40a58c307dae is now active with session: 2b11eab5-bc61-44bd-8216-a5695656c94c
Sep 22 23:21:38.156 INFO 5e3310b1-e4cc-48c6-92c2-40a58c307dae Set Active after no repair
Sep 22 23:21:38.156 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.156 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.156 INFO Set check for repair
Sep 22 23:21:38.156 INFO [1] 127.0.0.1:44178 task reports connection:true
Sep 22 23:21:38.156 INFO Created new region file "/tmp/downstairs-aIhbz7uq/region.json"
Sep 22 23:21:38.156 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.156 INFO 5e3310b1-e4cc-48c6-92c2-40a58c307dae Active Active Active
Sep 22 23:21:38.156 INFO Opened existing region file "/tmp/downstairs-IDeF0yuc/region.json"
Sep 22 23:21:38.156 INFO Set check for repair
Sep 22 23:21:38.156 INFO Database read version 1
Sep 22 23:21:38.156 INFO Database write version 1
Sep 22 23:21:38.156 INFO [2] 127.0.0.1:53578 task reports connection:true
Sep 22 23:21:38.156 INFO 5e3310b1-e4cc-48c6-92c2-40a58c307dae Active Active Active
Sep 22 23:21:38.156 INFO Set check for repair
Sep 22 23:21:38.156 INFO [0] received reconcile message
Sep 22 23:21:38.157 INFO [0] All repairs completed, exit
Sep 22 23:21:38.157 INFO [0] Starts cmd_loop
Sep 22 23:21:38.157 INFO [1] received reconcile message
Sep 22 23:21:38.157 INFO [1] All repairs completed, exit
Sep 22 23:21:38.157 INFO UUID: 9471030d-cea7-4287-81a6-b07789df3e4d
Sep 22 23:21:38.157 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.157 INFO [1] Starts cmd_loop
Sep 22 23:21:38.157 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.157 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.157 INFO [2] received reconcile message
Sep 22 23:21:38.157 INFO Using address: 127.0.0.1:52312, task: main
Sep 22 23:21:38.157 INFO [2] All repairs completed, exit
Sep 22 23:21:38.157 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.157 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.157 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39188, task: repair
Sep 22 23:21:38.157 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39188, task: repair
Sep 22 23:21:38.157 INFO listening, local_addr: 127.0.0.1:39188, task: repair
The guest has requested activation
Sep 22 23:21:38.157 INFO e89b9f37-20f1-48ba-8434-b618ba5c23aa active request set
Sep 22 23:21:38.157 INFO [0] received activate with gen 1
Sep 22 23:21:38.157 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39188, task: repair
Sep 22 23:21:38.157 INFO [0] client got ds_active_rx, promote! session 902609d5-e2d5-4264-985b-fc9832ca6874
Sep 22 23:21:38.157 INFO Using repair address: 127.0.0.1:39188, task: main
Sep 22 23:21:38.157 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.157 INFO [1] received activate with gen 1
Sep 22 23:21:38.157 INFO [1] client got ds_active_rx, promote! session 902609d5-e2d5-4264-985b-fc9832ca6874
Sep 22 23:21:38.157 INFO [2] received activate with gen 1
Sep 22 23:21:38.158 INFO [2] client got ds_active_rx, promote! session 902609d5-e2d5-4264-985b-fc9832ca6874
Sep 22 23:21:38.158 INFO UpstairsConnection { upstairs_id: e89b9f37-20f1-48ba-8434-b618ba5c23aa, session_id: 902609d5-e2d5-4264-985b-fc9832ca6874, gen: 1 } is now active (read-write)
Sep 22 23:21:38.158 INFO UpstairsConnection { upstairs_id: e89b9f37-20f1-48ba-8434-b618ba5c23aa, session_id: 902609d5-e2d5-4264-985b-fc9832ca6874, gen: 1 } is now active (read-write)
Sep 22 23:21:38.158 INFO UpstairsConnection { upstairs_id: e89b9f37-20f1-48ba-8434-b618ba5c23aa, session_id: 902609d5-e2d5-4264-985b-fc9832ca6874, gen: 1 } is now active (read-write)
Sep 22 23:21:38.158 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.158 INFO Created new region file "/tmp/downstairs-z98mSOht/region.json"
Sep 22 23:21:38.158 INFO [0] downstairs client at 127.0.0.1:58376 has UUID 481d7429-1224-42e4-a2f2-287bd0cfeaaf
Sep 22 23:21:38.158 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 481d7429-1224-42e4-a2f2-287bd0cfeaaf, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.158 INFO e89b9f37-20f1-48ba-8434-b618ba5c23aa WaitActive WaitActive WaitActive
Sep 22 23:21:38.158 INFO [1] downstairs client at 127.0.0.1:47267 has UUID f6750c55-7697-440f-81c9-8c2e90dcc4ce
Sep 22 23:21:38.158 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f6750c55-7697-440f-81c9-8c2e90dcc4ce, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.159 INFO e89b9f37-20f1-48ba-8434-b618ba5c23aa WaitActive WaitActive WaitActive
Sep 22 23:21:38.159 INFO [2] downstairs client at 127.0.0.1:54868 has UUID 4bd08688-0a92-4b76-9dcc-9afb5f6f7cc8
Sep 22 23:21:38.159 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4bd08688-0a92-4b76-9dcc-9afb5f6f7cc8, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.159 INFO e89b9f37-20f1-48ba-8434-b618ba5c23aa WaitActive WaitActive WaitActive
Sep 22 23:21:38.159 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.159 INFO UUID: c9ee3b95-925e-4745-a37e-7d93b0c64736
Sep 22 23:21:38.159 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.159 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.159 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.159 INFO Using address: 127.0.0.1:46844, task: main
Sep 22 23:21:38.159 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.159 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.160 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.160 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44127, task: repair
Sep 22 23:21:38.160 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44127, task: repair
Sep 22 23:21:38.160 INFO listening, local_addr: 127.0.0.1:44127, task: repair
Sep 22 23:21:38.160 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.160 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44127, task: repair
Sep 22 23:21:38.160 INFO Using repair address: 127.0.0.1:44127, task: main
Sep 22 23:21:38.160 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.160 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.160 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.160 INFO [0] e89b9f37-20f1-48ba-8434-b618ba5c23aa (902609d5-e2d5-4264-985b-fc9832ca6874) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.160 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.160 WARN [0] new RM replaced this: None
Sep 22 23:21:38.160 INFO [0] Starts reconcile loop
Sep 22 23:21:38.160 INFO [1] e89b9f37-20f1-48ba-8434-b618ba5c23aa (902609d5-e2d5-4264-985b-fc9832ca6874) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.160 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.160 WARN [1] new RM replaced this: None
Sep 22 23:21:38.160 INFO [1] Starts reconcile loop
Sep 22 23:21:38.160 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.160 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.160 INFO [2] e89b9f37-20f1-48ba-8434-b618ba5c23aa (902609d5-e2d5-4264-985b-fc9832ca6874) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.160 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.160 INFO Opened existing region file "/tmp/downstairs-aIhbz7uq/region.json"
Sep 22 23:21:38.160 WARN [2] new RM replaced this: None
Sep 22 23:21:38.160 INFO Database read version 1
Sep 22 23:21:38.160 INFO Database write version 1
Sep 22 23:21:38.160 INFO [2] Starts reconcile loop
Sep 22 23:21:38.161 INFO Created new region file "/tmp/downstairs-zWh48QwP/region.json"
Sep 22 23:21:38.161 INFO [0] 127.0.0.1:58376 task reports connection:true
Sep 22 23:21:38.161 INFO e89b9f37-20f1-48ba-8434-b618ba5c23aa WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.161 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.161 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.161 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.161 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.161 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.161 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.161 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.161 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.161 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.161 INFO Max found gen is 1
Sep 22 23:21:38.161 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.161 INFO Next flush: 1
Sep 22 23:21:38.161 INFO All extents match
Sep 22 23:21:38.161 INFO No downstairs repair required
Sep 22 23:21:38.161 INFO No initial repair work was required
Sep 22 23:21:38.161 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.161 INFO e89b9f37-20f1-48ba-8434-b618ba5c23aa is now active with session: 902609d5-e2d5-4264-985b-fc9832ca6874
Sep 22 23:21:38.161 INFO e89b9f37-20f1-48ba-8434-b618ba5c23aa Set Active after no repair
Sep 22 23:21:38.161 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.161 INFO Set check for repair
Sep 22 23:21:38.161 INFO [1] 127.0.0.1:47267 task reports connection:true
Sep 22 23:21:38.161 INFO e89b9f37-20f1-48ba-8434-b618ba5c23aa Active Active Active
Sep 22 23:21:38.161 INFO Set check for repair
Sep 22 23:21:38.161 INFO [2] 127.0.0.1:54868 task reports connection:true
Sep 22 23:21:38.161 INFO e89b9f37-20f1-48ba-8434-b618ba5c23aa Active Active Active
Sep 22 23:21:38.161 INFO Set check for repair
Sep 22 23:21:38.161 INFO [0] received reconcile message
Sep 22 23:21:38.161 INFO [0] All repairs completed, exit
Sep 22 23:21:38.161 INFO [0] Starts cmd_loop
Sep 22 23:21:38.161 INFO [1] received reconcile message
Sep 22 23:21:38.161 INFO [1] All repairs completed, exit
Sep 22 23:21:38.161 INFO [1] Starts cmd_loop
Sep 22 23:21:38.161 INFO [2] received reconcile message
Sep 22 23:21:38.161 INFO [2] All repairs completed, exit
Sep 22 23:21:38.161 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.163 INFO UUID: 89e24c32-e4e6-442a-a2a4-07d6620fd2c7
Sep 22 23:21:38.163 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.163 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.163 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.163 INFO Using address: 127.0.0.1:37306, task: main
Sep 22 23:21:38.163 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.163 INFO Opened existing region file "/tmp/downstairs-z98mSOht/region.json"
Sep 22 23:21:38.163 INFO Database read version 1
Sep 22 23:21:38.163 INFO Database write version 1
Sep 22 23:21:38.163 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.163 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.163 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51785, task: repair
Sep 22 23:21:38.163 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51785, task: repair
Sep 22 23:21:38.163 INFO listening, local_addr: 127.0.0.1:51785, task: repair
Sep 22 23:21:38.163 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51785, task: repair
Sep 22 23:21:38.163 INFO Using repair address: 127.0.0.1:51785, task: main
Sep 22 23:21:38.163 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
Sep 22 23:21:38.165 INFO UUID: 3c399f96-6189-45bc-9c78-28fb3e21a0fa
Sep 22 23:21:38.166 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.166 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.166 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.166 INFO Using address: 127.0.0.1:58283, task: main
Sep 22 23:21:38.166 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.166 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40584, task: repair
Sep 22 23:21:38.166 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40584, task: repair
Sep 22 23:21:38.166 INFO listening, local_addr: 127.0.0.1:40584, task: repair
Sep 22 23:21:38.166 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.166 INFO Opened existing region file "/tmp/downstairs-zWh48QwP/region.json"
Sep 22 23:21:38.166 INFO Database read version 1
Sep 22 23:21:38.166 INFO Database write version 1
Sep 22 23:21:38.166 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40584, task: repair
Sep 22 23:21:38.166 INFO Using repair address: 127.0.0.1:40584, task: main
Sep 22 23:21:38.166 INFO No SSL acceptor configured, task: main
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.166865288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:51585","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:51585","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.166949671Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:38.167 INFO Upstairs starts
Sep 22 23:21:38.167 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.167 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.167 INFO Crucible stats registered with UUID: 921a859f-fc02-4985-a3ec-da920cfa511a
Sep 22 23:21:38.167 INFO Crucible 921a859f-fc02-4985-a3ec-da920cfa511a has session id: 25183a0b-04a2-464c-b2a4-ba6ae86fb122
Sep 22 23:21:38.167 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.167 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.167 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.167 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.168 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.168 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.168 INFO [0] connecting to 127.0.0.1:51219, looper: 0
Sep 22 23:21:38.168 INFO [1] connecting to 127.0.0.1:44324, looper: 1
Sep 22 23:21:38.168 INFO [2] connecting to 127.0.0.1:52939, looper: 2
Sep 22 23:21:38.168 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.168 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.168 INFO Flush timeout: 0.5
Sep 22 23:21:38.168 INFO accepted connection from 127.0.0.1:47115, task: main
Sep 22 23:21:38.168 INFO accepted connection from 127.0.0.1:55027, task: main
Sep 22 23:21:38.168 INFO accepted connection from 127.0.0.1:44125, task: main
Sep 22 23:21:38.168 INFO [0] 921a859f-fc02-4985-a3ec-da920cfa511a looper connected, looper: 0
Sep 22 23:21:38.168 INFO [0] Proc runs for 127.0.0.1:51219 in state New
Sep 22 23:21:38.168 INFO [1] 921a859f-fc02-4985-a3ec-da920cfa511a looper connected, looper: 1
Sep 22 23:21:38.168 INFO [1] Proc runs for 127.0.0.1:44324 in state New
Sep 22 23:21:38.168 INFO [2] 921a859f-fc02-4985-a3ec-da920cfa511a looper connected, looper: 2
Sep 22 23:21:38.169 INFO [2] Proc runs for 127.0.0.1:52939 in state New
Sep 22 23:21:38.169 DEBG up_ds_listen was notified
Sep 22 23:21:38.169 INFO UUID: 2d5bbbc4-fa02-4480-b887-9b6f4980eb1b
Sep 22 23:21:38.169 DEBG up_ds_listen process 1000
Sep 22 23:21:38.169 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.169 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.169 INFO Upstairs starts
Sep 22 23:21:38.169 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.169 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.169 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.169 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.169 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.169 INFO Crucible stats registered with UUID: 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa
Sep 22 23:21:38.169 INFO Using address: 127.0.0.1:39258, task: main
Sep 22 23:21:38.169 INFO Crucible 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa has session id: fc30cbc3-86fb-4d4d-b35d-cb789cc9bc9f
Sep 22 23:21:38.169 INFO Connection request from 921a859f-fc02-4985-a3ec-da920cfa511a with version 4, task: proc
Sep 22 23:21:38.169 INFO upstairs UpstairsConnection { upstairs_id: 921a859f-fc02-4985-a3ec-da920cfa511a, session_id: 00e59270-0ce2-4dbd-bb99-47e86c7cfd3e, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.169 INFO Connection request from 921a859f-fc02-4985-a3ec-da920cfa511a with version 4, task: proc
Sep 22 23:21:38.169 INFO upstairs UpstairsConnection { upstairs_id: 921a859f-fc02-4985-a3ec-da920cfa511a, session_id: 00e59270-0ce2-4dbd-bb99-47e86c7cfd3e, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.169 INFO Connection request from 921a859f-fc02-4985-a3ec-da920cfa511a with version 4, task: proc
Sep 22 23:21:38.169 INFO upstairs UpstairsConnection { upstairs_id: 921a859f-fc02-4985-a3ec-da920cfa511a, session_id: 00e59270-0ce2-4dbd-bb99-47e86c7cfd3e, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.169 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.169 INFO [0] connecting to 127.0.0.1:48752, looper: 0
Sep 22 23:21:38.169 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:61391, task: repair
Sep 22 23:21:38.169 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:61391, task: repair
Sep 22 23:21:38.169 INFO [1] connecting to 127.0.0.1:52312, looper: 1
Sep 22 23:21:38.169 INFO listening, local_addr: 127.0.0.1:61391, task: repair
Sep 22 23:21:38.170 INFO [2] connecting to 127.0.0.1:58283, looper: 2
Sep 22 23:21:38.170 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:61391, task: repair
Sep 22 23:21:38.170 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.170 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.170 INFO Flush timeout: 0.5
Sep 22 23:21:38.170 INFO Using repair address: 127.0.0.1:61391, task: main
Sep 22 23:21:38.170 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.170 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.170 INFO accepted connection from 127.0.0.1:46328, task: main
Sep 22 23:21:38.170 INFO Created new region file "/tmp/downstairs-lFV1uaQD/region.json"
Sep 22 23:21:38.170 INFO accepted connection from 127.0.0.1:55292, task: main
Sep 22 23:21:38.170 INFO accepted connection from 127.0.0.1:51427, task: main
Sep 22 23:21:38.170 INFO [0] 921a859f-fc02-4985-a3ec-da920cfa511a (00e59270-0ce2-4dbd-bb99-47e86c7cfd3e) New New New ds_transition to WaitActive
Sep 22 23:21:38.170 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.170 INFO [1] 921a859f-fc02-4985-a3ec-da920cfa511a (00e59270-0ce2-4dbd-bb99-47e86c7cfd3e) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.170 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.170 INFO [2] 921a859f-fc02-4985-a3ec-da920cfa511a (00e59270-0ce2-4dbd-bb99-47e86c7cfd3e) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.170 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.170 INFO [0] 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa looper connected, looper: 0
Sep 22 23:21:38.171 INFO [0] Proc runs for 127.0.0.1:48752 in state New
Sep 22 23:21:38.171 INFO [1] 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa looper connected, looper: 1
Sep 22 23:21:38.171 INFO [1] Proc runs for 127.0.0.1:52312 in state New
Sep 22 23:21:38.171 INFO [2] 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa looper connected, looper: 2
Sep 22 23:21:38.171 INFO [2] Proc runs for 127.0.0.1:58283 in state New
Sep 22 23:21:38.171 INFO Connection request from 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa with version 4, task: proc
Sep 22 23:21:38.171 INFO upstairs UpstairsConnection { upstairs_id: 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa, session_id: eb0da1b5-c89a-40f6-a24b-5a6a9adb2329, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.171 INFO Connection request from 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa with version 4, task: proc
Sep 22 23:21:38.171 INFO upstairs UpstairsConnection { upstairs_id: 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa, session_id: eb0da1b5-c89a-40f6-a24b-5a6a9adb2329, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.171 INFO Connection request from 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa with version 4, task: proc
Sep 22 23:21:38.171 INFO upstairs UpstairsConnection { upstairs_id: 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa, session_id: eb0da1b5-c89a-40f6-a24b-5a6a9adb2329, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.172 INFO [0] 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa (eb0da1b5-c89a-40f6-a24b-5a6a9adb2329) New New New ds_transition to WaitActive
Sep 22 23:21:38.172 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.172 INFO [1] 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa (eb0da1b5-c89a-40f6-a24b-5a6a9adb2329) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.172 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.172 INFO [2] 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa (eb0da1b5-c89a-40f6-a24b-5a6a9adb2329) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.172 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.172 DEBG IO Write 1000 has deps []
The guest has requested activation
Sep 22 23:21:38.172 INFO 921a859f-fc02-4985-a3ec-da920cfa511a active request set
Sep 22 23:21:38.172 INFO [0] received activate with gen 1
Sep 22 23:21:38.172 INFO [0] client got ds_active_rx, promote! session 00e59270-0ce2-4dbd-bb99-47e86c7cfd3e
Sep 22 23:21:38.172 INFO [1] received activate with gen 1
Sep 22 23:21:38.172 INFO [1] client got ds_active_rx, promote! session 00e59270-0ce2-4dbd-bb99-47e86c7cfd3e
Sep 22 23:21:38.172 INFO [2] received activate with gen 1
Sep 22 23:21:38.172 INFO [2] client got ds_active_rx, promote! session 00e59270-0ce2-4dbd-bb99-47e86c7cfd3e
Sep 22 23:21:38.172 INFO UpstairsConnection { upstairs_id: 921a859f-fc02-4985-a3ec-da920cfa511a, session_id: 00e59270-0ce2-4dbd-bb99-47e86c7cfd3e, gen: 1 } is now active (read-write)
Sep 22 23:21:38.173 INFO UpstairsConnection { upstairs_id: 921a859f-fc02-4985-a3ec-da920cfa511a, session_id: 00e59270-0ce2-4dbd-bb99-47e86c7cfd3e, gen: 1 } is now active (read-write)
Sep 22 23:21:38.173 INFO UpstairsConnection { upstairs_id: 921a859f-fc02-4985-a3ec-da920cfa511a, session_id: 00e59270-0ce2-4dbd-bb99-47e86c7cfd3e, gen: 1 } is now active (read-write)
Sep 22 23:21:38.173 INFO [0] downstairs client at 127.0.0.1:51219 has UUID 71f89d71-fca2-4667-8b12-c125e7772d19
Sep 22 23:21:38.173 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 71f89d71-fca2-4667-8b12-c125e7772d19, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.173 INFO 921a859f-fc02-4985-a3ec-da920cfa511a WaitActive WaitActive WaitActive
Sep 22 23:21:38.173 INFO [1] downstairs client at 127.0.0.1:44324 has UUID 7c9decb7-dee6-4a23-a2b7-7860a60db1d6
Sep 22 23:21:38.173 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 7c9decb7-dee6-4a23-a2b7-7860a60db1d6, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.173 INFO 921a859f-fc02-4985-a3ec-da920cfa511a WaitActive WaitActive WaitActive
Sep 22 23:21:38.174 INFO [2] downstairs client at 127.0.0.1:52939 has UUID 6e4f126c-20d7-47c5-9d94-936338c4e38d
Sep 22 23:21:38.174 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6e4f126c-20d7-47c5-9d94-936338c4e38d, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.174 INFO 921a859f-fc02-4985-a3ec-da920cfa511a WaitActive WaitActive WaitActive
Sep 22 23:21:38.174 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.174 INFO Opened existing region file "/tmp/downstairs-lFV1uaQD/region.json"
Sep 22 23:21:38.174 INFO Database read version 1
Sep 22 23:21:38.174 INFO Database write version 1
Sep 22 23:21:38.174 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.174 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.174 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.174 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.175 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.175 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.175 INFO [0] 921a859f-fc02-4985-a3ec-da920cfa511a (00e59270-0ce2-4dbd-bb99-47e86c7cfd3e) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.175 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.175 WARN [0] new RM replaced this: None
Sep 22 23:21:38.175 INFO [0] Starts reconcile loop
Sep 22 23:21:38.175 INFO [1] 921a859f-fc02-4985-a3ec-da920cfa511a (00e59270-0ce2-4dbd-bb99-47e86c7cfd3e) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.175 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.175 WARN [1] new RM replaced this: None
Sep 22 23:21:38.175 INFO [1] Starts reconcile loop
Sep 22 23:21:38.175 INFO [2] 921a859f-fc02-4985-a3ec-da920cfa511a (00e59270-0ce2-4dbd-bb99-47e86c7cfd3e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.175 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.175 WARN [2] new RM replaced this: None
Sep 22 23:21:38.175 INFO [2] Starts reconcile loop
Sep 22 23:21:38.176 INFO [0] 127.0.0.1:51219 task reports connection:true
Sep 22 23:21:38.176 INFO 921a859f-fc02-4985-a3ec-da920cfa511a WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.176 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.176 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.176 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.176 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.176 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.176 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.176 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.176 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.176 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.176 INFO Max found gen is 1
Sep 22 23:21:38.176 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.176 INFO Next flush: 1
Sep 22 23:21:38.176 INFO All extents match
Sep 22 23:21:38.176 INFO No downstairs repair required
Sep 22 23:21:38.176 INFO No initial repair work was required
Sep 22 23:21:38.176 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.176 INFO 921a859f-fc02-4985-a3ec-da920cfa511a is now active with session: 00e59270-0ce2-4dbd-bb99-47e86c7cfd3e
Sep 22 23:21:38.176 INFO 921a859f-fc02-4985-a3ec-da920cfa511a Set Active after no repair
Sep 22 23:21:38.176 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.176 INFO Set check for repair
Sep 22 23:21:38.176 INFO [1] 127.0.0.1:44324 task reports connection:true
Sep 22 23:21:38.176 INFO 921a859f-fc02-4985-a3ec-da920cfa511a Active Active Active
Sep 22 23:21:38.176 INFO Set check for repair
Sep 22 23:21:38.176 INFO [2] 127.0.0.1:52939 task reports connection:true
Sep 22 23:21:38.176 INFO 921a859f-fc02-4985-a3ec-da920cfa511a Active Active Active
Sep 22 23:21:38.176 INFO Set check for repair
Sep 22 23:21:38.176 INFO [0] received reconcile message
Sep 22 23:21:38.176 INFO [0] All repairs completed, exit
Sep 22 23:21:38.176 INFO [0] Starts cmd_loop
Sep 22 23:21:38.176 INFO UUID: 635f8034-f948-4642-a596-c4452884ff7b
Sep 22 23:21:38.176 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.176 INFO [1] received reconcile message
Sep 22 23:21:38.176 INFO [1] All repairs completed, exit
Sep 22 23:21:38.176 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.177 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.177 INFO [1] Starts cmd_loop
Sep 22 23:21:38.177 INFO Using address: 127.0.0.1:63407, task: main
Sep 22 23:21:38.177 INFO [2] received reconcile message
Sep 22 23:21:38.177 INFO [2] All repairs completed, exit
Sep 22 23:21:38.177 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.177 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.177 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62799, task: repair
Sep 22 23:21:38.177 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62799, task: repair
The guest has requested activation
Sep 22 23:21:38.177 INFO listening, local_addr: 127.0.0.1:62799, task: repair
Sep 22 23:21:38.177 INFO 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa active request set
Sep 22 23:21:38.177 INFO [0] received activate with gen 1
Sep 22 23:21:38.177 INFO [0] client got ds_active_rx, promote! session eb0da1b5-c89a-40f6-a24b-5a6a9adb2329
Sep 22 23:21:38.177 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62799, task: repair
Sep 22 23:21:38.177 INFO Using repair address: 127.0.0.1:62799, task: main
Sep 22 23:21:38.177 INFO [1] received activate with gen 1
Sep 22 23:21:38.177 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.177 INFO [1] client got ds_active_rx, promote! session eb0da1b5-c89a-40f6-a24b-5a6a9adb2329
Sep 22 23:21:38.178 INFO [2] received activate with gen 1
Sep 22 23:21:38.178 INFO [2] client got ds_active_rx, promote! session eb0da1b5-c89a-40f6-a24b-5a6a9adb2329
Sep 22 23:21:38.178 INFO UpstairsConnection { upstairs_id: 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa, session_id: eb0da1b5-c89a-40f6-a24b-5a6a9adb2329, gen: 1 } is now active (read-write)
Sep 22 23:21:38.178 INFO UpstairsConnection { upstairs_id: 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa, session_id: eb0da1b5-c89a-40f6-a24b-5a6a9adb2329, gen: 1 } is now active (read-write)
Sep 22 23:21:38.178 DEBG up_ds_listen was notified
Sep 22 23:21:38.178 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.178 DEBG up_ds_listen process 1000
Sep 22 23:21:38.178 INFO UpstairsConnection { upstairs_id: 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa, session_id: eb0da1b5-c89a-40f6-a24b-5a6a9adb2329, gen: 1 } is now active (read-write)
Sep 22 23:21:38.178 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.178 INFO Created new region file "/tmp/downstairs-BulvGBTg/region.json"
Sep 22 23:21:38.178 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.178 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:21:38.178 INFO [0] downstairs client at 127.0.0.1:48752 has UUID 0d1968cb-b30f-4a44-a680-9f3b52cfb45a
Sep 22 23:21:38.178 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 0d1968cb-b30f-4a44-a680-9f3b52cfb45a, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.179 INFO 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa WaitActive WaitActive WaitActive
Sep 22 23:21:38.179 INFO [1] downstairs client at 127.0.0.1:52312 has UUID 9471030d-cea7-4287-81a6-b07789df3e4d
Sep 22 23:21:38.179 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9471030d-cea7-4287-81a6-b07789df3e4d, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.179 INFO 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa WaitActive WaitActive WaitActive
Sep 22 23:21:38.179 INFO [2] downstairs client at 127.0.0.1:58283 has UUID 3c399f96-6189-45bc-9c78-28fb3e21a0fa
Sep 22 23:21:38.179 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 3c399f96-6189-45bc-9c78-28fb3e21a0fa, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.179 INFO 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa WaitActive WaitActive WaitActive
Sep 22 23:21:38.179 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.179 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.179 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.180 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.180 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.180 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.180 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.180 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.180 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.181 INFO [0] 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa (eb0da1b5-c89a-40f6-a24b-5a6a9adb2329) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.181 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.181 WARN [0] new RM replaced this: None
Sep 22 23:21:38.181 INFO [0] Starts reconcile loop
Sep 22 23:21:38.181 INFO [1] 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa (eb0da1b5-c89a-40f6-a24b-5a6a9adb2329) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.181 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.181 WARN [1] new RM replaced this: None
Sep 22 23:21:38.181 INFO [1] Starts reconcile loop
Sep 22 23:21:38.181 INFO [2] 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa (eb0da1b5-c89a-40f6-a24b-5a6a9adb2329) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.181 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.181 WARN [2] new RM replaced this: None
Sep 22 23:21:38.181 INFO [2] Starts reconcile loop
Sep 22 23:21:38.181 INFO [0] 127.0.0.1:48752 task reports connection:true
Sep 22 23:21:38.181 INFO 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.181 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.181 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.181 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.181 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.181 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.181 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.181 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.181 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.181 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.181 INFO Max found gen is 1
Sep 22 23:21:38.181 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.181 INFO Next flush: 1
Sep 22 23:21:38.181 INFO All extents match
Sep 22 23:21:38.181 INFO No downstairs repair required
Sep 22 23:21:38.181 INFO No initial repair work was required
Sep 22 23:21:38.181 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.181 INFO 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa is now active with session: eb0da1b5-c89a-40f6-a24b-5a6a9adb2329
Sep 22 23:21:38.181 INFO 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa Set Active after no repair
Sep 22 23:21:38.181 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.181 INFO Set check for repair
Sep 22 23:21:38.181 INFO [1] 127.0.0.1:52312 task reports connection:true
Sep 22 23:21:38.182 INFO 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa Active Active Active
Sep 22 23:21:38.182 INFO Set check for repair
Sep 22 23:21:38.182 INFO [2] 127.0.0.1:58283 task reports connection:true
Sep 22 23:21:38.182 INFO 5bb4c0ba-2389-4fe1-a4cc-d740e8a65daa Active Active Active
Sep 22 23:21:38.182 INFO Set check for repair
Sep 22 23:21:38.182 INFO [0] received reconcile message
Sep 22 23:21:38.182 INFO [0] All repairs completed, exit
Sep 22 23:21:38.182 INFO [0] Starts cmd_loop
Sep 22 23:21:38.182 INFO [1] received reconcile message
Sep 22 23:21:38.182 INFO [1] All repairs completed, exit
Sep 22 23:21:38.182 INFO [1] Starts cmd_loop
Sep 22 23:21:38.182 INFO [2] received reconcile message
Sep 22 23:21:38.182 INFO [2] All repairs completed, exit
Sep 22 23:21:38.182 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.183 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.183 INFO Opened existing region file "/tmp/downstairs-BulvGBTg/region.json"
Sep 22 23:21:38.183 INFO Database read version 1
Sep 22 23:21:38.183 INFO Database write version 1
Sep 22 23:21:38.183 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:38.183 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.184 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.185 INFO UUID: d6053127-e474-4389-afe6-5018c4a351d2
Sep 22 23:21:38.185 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.185 DEBG up_ds_listen was notified
Sep 22 23:21:38.185 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.185 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.185 DEBG up_ds_listen process 1000
Sep 22 23:21:38.185 INFO Using address: 127.0.0.1:42114, task: main
Sep 22 23:21:38.185 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.185 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.185 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.186 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49825, task: repair
Sep 22 23:21:38.186 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49825, task: repair
Sep 22 23:21:38.186 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.186 INFO listening, local_addr: 127.0.0.1:49825, task: repair
Sep 22 23:21:38.186 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49825, task: repair
Sep 22 23:21:38.186 INFO Using repair address: 127.0.0.1:49825, task: main
Sep 22 23:21:38.186 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.186 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.186 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.186 INFO Opened existing region file "/tmp/downstairs-wZJpXBn2/region.json"
Sep 22 23:21:38.186 INFO Database read version 1
Sep 22 23:21:38.186 INFO Database write version 1
Sep 22 23:21:38.186 DEBG up_ds_listen was notified
Sep 22 23:21:38.186 DEBG up_ds_listen process 1001
Sep 22 23:21:38.186 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.186 INFO Upstairs starts
Sep 22 23:21:38.186 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.186 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.186 INFO Crucible stats registered with UUID: 913a7ee4-ee11-47c7-a343-a7c9547130eb
Sep 22 23:21:38.186 INFO Crucible 913a7ee4-ee11-47c7-a343-a7c9547130eb has session id: b580f6b1-5b80-4eb2-9b2e-ec0df1111077
Sep 22 23:21:38.187 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.187 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.187 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.187 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.187 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.187 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.187 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.187 INFO [0] connecting to 127.0.0.1:62388, looper: 0
Sep 22 23:21:38.187 INFO [1] connecting to 127.0.0.1:43611, looper: 1
Sep 22 23:21:38.187 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:21:38.187 INFO [2] connecting to 127.0.0.1:46844, looper: 2
Sep 22 23:21:38.187 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.187 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.187 INFO Flush timeout: 0.5
Sep 22 23:21:38.188 INFO accepted connection from 127.0.0.1:39854, task: main
Sep 22 23:21:38.188 INFO accepted connection from 127.0.0.1:54255, task: main
Sep 22 23:21:38.188 INFO accepted connection from 127.0.0.1:57006, task: main
Sep 22 23:21:38.188 INFO [0] 913a7ee4-ee11-47c7-a343-a7c9547130eb looper connected, looper: 0
Sep 22 23:21:38.188 INFO [0] Proc runs for 127.0.0.1:62388 in state New
Sep 22 23:21:38.188 INFO [1] 913a7ee4-ee11-47c7-a343-a7c9547130eb looper connected, looper: 1
Sep 22 23:21:38.188 INFO [1] Proc runs for 127.0.0.1:43611 in state New
Sep 22 23:21:38.188 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.188 INFO [2] 913a7ee4-ee11-47c7-a343-a7c9547130eb looper connected, looper: 2
Sep 22 23:21:38.188 DEBG up_ds_listen was notified
Sep 22 23:21:38.188 INFO [2] Proc runs for 127.0.0.1:46844 in state New
Sep 22 23:21:38.188 DEBG up_ds_listen process 1000
Sep 22 23:21:38.188 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.188 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.188 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.188 INFO Upstairs starts
Sep 22 23:21:38.188 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.189 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.189 INFO Crucible stats registered with UUID: edae0731-d549-4a19-a15b-e1c9f01172b4
Sep 22 23:21:38.189 INFO Crucible edae0731-d549-4a19-a15b-e1c9f01172b4 has session id: cab8b72b-236e-4a56-8922-0a546f70b336
Sep 22 23:21:38.189 INFO Connection request from 913a7ee4-ee11-47c7-a343-a7c9547130eb with version 4, task: proc
Sep 22 23:21:38.189 INFO upstairs UpstairsConnection { upstairs_id: 913a7ee4-ee11-47c7-a343-a7c9547130eb, session_id: fbf121c3-eef6-4f40-8f3c-e3ef373940ac, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.189 INFO Connection request from 913a7ee4-ee11-47c7-a343-a7c9547130eb with version 4, task: proc
Sep 22 23:21:38.189 INFO upstairs UpstairsConnection { upstairs_id: 913a7ee4-ee11-47c7-a343-a7c9547130eb, session_id: fbf121c3-eef6-4f40-8f3c-e3ef373940ac, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.189 INFO Connection request from 913a7ee4-ee11-47c7-a343-a7c9547130eb with version 4, task: proc
Sep 22 23:21:38.189 INFO upstairs UpstairsConnection { upstairs_id: 913a7ee4-ee11-47c7-a343-a7c9547130eb, session_id: fbf121c3-eef6-4f40-8f3c-e3ef373940ac, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.189 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.189 INFO [0] connecting to 127.0.0.1:39258, looper: 0
Sep 22 23:21:38.189 INFO [1] connecting to 127.0.0.1:63407, looper: 1
Sep 22 23:21:38.189 INFO [2] connecting to 127.0.0.1:42114, looper: 2
Sep 22 23:21:38.189 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.189 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.189 INFO Flush timeout: 0.5
Sep 22 23:21:38.190 INFO accepted connection from 127.0.0.1:51279, task: main
Sep 22 23:21:38.190 INFO accepted connection from 127.0.0.1:63124, task: main
Sep 22 23:21:38.190 INFO accepted connection from 127.0.0.1:33283, task: main
Sep 22 23:21:38.190 INFO [0] 913a7ee4-ee11-47c7-a343-a7c9547130eb (fbf121c3-eef6-4f40-8f3c-e3ef373940ac) New New New ds_transition to WaitActive
Sep 22 23:21:38.190 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.190 INFO [1] 913a7ee4-ee11-47c7-a343-a7c9547130eb (fbf121c3-eef6-4f40-8f3c-e3ef373940ac) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.190 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.190 INFO [2] 913a7ee4-ee11-47c7-a343-a7c9547130eb (fbf121c3-eef6-4f40-8f3c-e3ef373940ac) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.190 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.190 INFO [0] edae0731-d549-4a19-a15b-e1c9f01172b4 looper connected, looper: 0
Sep 22 23:21:38.190 INFO [0] Proc runs for 127.0.0.1:39258 in state New
Sep 22 23:21:38.190 INFO [1] edae0731-d549-4a19-a15b-e1c9f01172b4 looper connected, looper: 1
Sep 22 23:21:38.190 INFO [1] Proc runs for 127.0.0.1:63407 in state New
Sep 22 23:21:38.190 INFO [2] edae0731-d549-4a19-a15b-e1c9f01172b4 looper connected, looper: 2
Sep 22 23:21:38.190 INFO [2] Proc runs for 127.0.0.1:42114 in state New
Sep 22 23:21:38.191 INFO Connection request from edae0731-d549-4a19-a15b-e1c9f01172b4 with version 4, task: proc
Sep 22 23:21:38.191 INFO upstairs UpstairsConnection { upstairs_id: edae0731-d549-4a19-a15b-e1c9f01172b4, session_id: d263b9b4-f13a-4d85-ac47-3dad10fd9290, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.191 INFO Connection request from edae0731-d549-4a19-a15b-e1c9f01172b4 with version 4, task: proc
Sep 22 23:21:38.191 INFO upstairs UpstairsConnection { upstairs_id: edae0731-d549-4a19-a15b-e1c9f01172b4, session_id: d263b9b4-f13a-4d85-ac47-3dad10fd9290, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.191 INFO Connection request from edae0731-d549-4a19-a15b-e1c9f01172b4 with version 4, task: proc
Sep 22 23:21:38.191 INFO upstairs UpstairsConnection { upstairs_id: edae0731-d549-4a19-a15b-e1c9f01172b4, session_id: d263b9b4-f13a-4d85-ac47-3dad10fd9290, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.191 INFO [0] edae0731-d549-4a19-a15b-e1c9f01172b4 (d263b9b4-f13a-4d85-ac47-3dad10fd9290) New New New ds_transition to WaitActive
Sep 22 23:21:38.191 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.191 INFO [1] edae0731-d549-4a19-a15b-e1c9f01172b4 (d263b9b4-f13a-4d85-ac47-3dad10fd9290) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.191 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.191 INFO [2] edae0731-d549-4a19-a15b-e1c9f01172b4 (d263b9b4-f13a-4d85-ac47-3dad10fd9290) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.191 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:21:38.191 INFO 913a7ee4-ee11-47c7-a343-a7c9547130eb active request set
Sep 22 23:21:38.191 INFO [0] received activate with gen 1
Sep 22 23:21:38.191 INFO [0] client got ds_active_rx, promote! session fbf121c3-eef6-4f40-8f3c-e3ef373940ac
Sep 22 23:21:38.191 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:38.191 INFO [1] received activate with gen 1
Sep 22 23:21:38.191 INFO [1] client got ds_active_rx, promote! session fbf121c3-eef6-4f40-8f3c-e3ef373940ac
Sep 22 23:21:38.191 INFO [2] received activate with gen 1
Sep 22 23:21:38.191 INFO [2] client got ds_active_rx, promote! session fbf121c3-eef6-4f40-8f3c-e3ef373940ac
Sep 22 23:21:38.192 INFO UpstairsConnection { upstairs_id: 913a7ee4-ee11-47c7-a343-a7c9547130eb, session_id: fbf121c3-eef6-4f40-8f3c-e3ef373940ac, gen: 1 } is now active (read-write)
Sep 22 23:21:38.192 INFO UpstairsConnection { upstairs_id: 913a7ee4-ee11-47c7-a343-a7c9547130eb, session_id: fbf121c3-eef6-4f40-8f3c-e3ef373940ac, gen: 1 } is now active (read-write)
Sep 22 23:21:38.192 INFO UpstairsConnection { upstairs_id: 913a7ee4-ee11-47c7-a343-a7c9547130eb, session_id: fbf121c3-eef6-4f40-8f3c-e3ef373940ac, gen: 1 } is now active (read-write)
Sep 22 23:21:38.192 INFO [0] downstairs client at 127.0.0.1:62388 has UUID 6a3090e7-a0f2-4a80-947f-1db172a31a5f
Sep 22 23:21:38.192 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6a3090e7-a0f2-4a80-947f-1db172a31a5f, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.192 INFO 913a7ee4-ee11-47c7-a343-a7c9547130eb WaitActive WaitActive WaitActive
Sep 22 23:21:38.192 INFO [1] downstairs client at 127.0.0.1:43611 has UUID ab814286-bbb3-4f61-a468-0a3953430633
Sep 22 23:21:38.192 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ab814286-bbb3-4f61-a468-0a3953430633, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.192 INFO 913a7ee4-ee11-47c7-a343-a7c9547130eb WaitActive WaitActive WaitActive
Sep 22 23:21:38.192 INFO [2] downstairs client at 127.0.0.1:46844 has UUID c9ee3b95-925e-4745-a37e-7d93b0c64736
Sep 22 23:21:38.192 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c9ee3b95-925e-4745-a37e-7d93b0c64736, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.192 INFO 913a7ee4-ee11-47c7-a343-a7c9547130eb WaitActive WaitActive WaitActive
Sep 22 23:21:38.193 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.193 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.193 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.193 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.193 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.193 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.193 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:38.194 INFO [0] 913a7ee4-ee11-47c7-a343-a7c9547130eb (fbf121c3-eef6-4f40-8f3c-e3ef373940ac) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.194 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.194 WARN [0] new RM replaced this: None
Sep 22 23:21:38.194 INFO [0] Starts reconcile loop
Sep 22 23:21:38.194 INFO [1] 913a7ee4-ee11-47c7-a343-a7c9547130eb (fbf121c3-eef6-4f40-8f3c-e3ef373940ac) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.194 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.194 WARN [1] new RM replaced this: None
Sep 22 23:21:38.194 INFO [1] Starts reconcile loop
Sep 22 23:21:38.194 INFO [2] 913a7ee4-ee11-47c7-a343-a7c9547130eb (fbf121c3-eef6-4f40-8f3c-e3ef373940ac) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.194 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.194 WARN [2] new RM replaced this: None
Sep 22 23:21:38.194 INFO [2] Starts reconcile loop
Sep 22 23:21:38.194 INFO [0] 127.0.0.1:62388 task reports connection:true
Sep 22 23:21:38.194 INFO 913a7ee4-ee11-47c7-a343-a7c9547130eb WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.194 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.194 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.194 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.194 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.194 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.194 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.194 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.194 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.194 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.194 INFO Max found gen is 1
Sep 22 23:21:38.194 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.194 INFO Next flush: 1
Sep 22 23:21:38.194 INFO All extents match
Sep 22 23:21:38.194 INFO No downstairs repair required
Sep 22 23:21:38.194 INFO UUID: d1402af7-f301-4500-84d1-bcd26b1c483b
Sep 22 23:21:38.194 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:38.194 INFO No initial repair work was required
Sep 22 23:21:38.194 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.194 INFO 913a7ee4-ee11-47c7-a343-a7c9547130eb is now active with session: fbf121c3-eef6-4f40-8f3c-e3ef373940ac
Sep 22 23:21:38.194 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.194 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.194 INFO 913a7ee4-ee11-47c7-a343-a7c9547130eb Set Active after no repair
Sep 22 23:21:38.194 INFO Using address: 127.0.0.1:36172, task: main
Sep 22 23:21:38.194 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.194 INFO Set check for repair
Sep 22 23:21:38.195 INFO [1] 127.0.0.1:43611 task reports connection:true
Sep 22 23:21:38.195 INFO 913a7ee4-ee11-47c7-a343-a7c9547130eb Active Active Active
Sep 22 23:21:38.195 INFO Set check for repair
Sep 22 23:21:38.195 INFO [2] 127.0.0.1:46844 task reports connection:true
Sep 22 23:21:38.195 INFO 913a7ee4-ee11-47c7-a343-a7c9547130eb Active Active Active
Sep 22 23:21:38.195 INFO Set check for repair
Sep 22 23:21:38.195 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.195 INFO [0] received reconcile message
Sep 22 23:21:38.195 INFO [0] All repairs completed, exit
Sep 22 23:21:38.195 INFO [0] Starts cmd_loop
Sep 22 23:21:38.195 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44765, task: repair
Sep 22 23:21:38.195 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44765, task: repair
Sep 22 23:21:38.195 INFO [1] received reconcile message
Sep 22 23:21:38.195 INFO listening, local_addr: 127.0.0.1:44765, task: repair
Sep 22 23:21:38.195 INFO [1] All repairs completed, exit
Sep 22 23:21:38.195 INFO [1] Starts cmd_loop
Sep 22 23:21:38.195 INFO [2] received reconcile message
Sep 22 23:21:38.195 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44765, task: repair
Sep 22 23:21:38.195 INFO Using repair address: 127.0.0.1:44765, task: main
Sep 22 23:21:38.195 INFO [2] All repairs completed, exit
Sep 22 23:21:38.195 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.195 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
The guest has requested activation
Sep 22 23:21:38.196 INFO edae0731-d549-4a19-a15b-e1c9f01172b4 active request set
Sep 22 23:21:38.196 INFO [0] received activate with gen 1
Sep 22 23:21:38.196 INFO [0] client got ds_active_rx, promote! session d263b9b4-f13a-4d85-ac47-3dad10fd9290
Sep 22 23:21:38.196 INFO [1] received activate with gen 1
Sep 22 23:21:38.196 INFO [1] client got ds_active_rx, promote! session d263b9b4-f13a-4d85-ac47-3dad10fd9290
Sep 22 23:21:38.196 INFO [2] received activate with gen 1
Sep 22 23:21:38.196 INFO [2] client got ds_active_rx, promote! session d263b9b4-f13a-4d85-ac47-3dad10fd9290
Sep 22 23:21:38.196 INFO UpstairsConnection { upstairs_id: edae0731-d549-4a19-a15b-e1c9f01172b4, session_id: d263b9b4-f13a-4d85-ac47-3dad10fd9290, gen: 1 } is now active (read-write)
Sep 22 23:21:38.196 INFO UpstairsConnection { upstairs_id: edae0731-d549-4a19-a15b-e1c9f01172b4, session_id: d263b9b4-f13a-4d85-ac47-3dad10fd9290, gen: 1 } is now active (read-write)
Sep 22 23:21:38.196 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.196 INFO UpstairsConnection { upstairs_id: edae0731-d549-4a19-a15b-e1c9f01172b4, session_id: d263b9b4-f13a-4d85-ac47-3dad10fd9290, gen: 1 } is now active (read-write)
Sep 22 23:21:38.196 DEBG up_ds_listen was notified
Sep 22 23:21:38.196 DEBG up_ds_listen process 1001
Sep 22 23:21:38.196 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.196 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:21:38.196 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.197 INFO [0] downstairs client at 127.0.0.1:39258 has UUID 2d5bbbc4-fa02-4480-b887-9b6f4980eb1b
Sep 22 23:21:38.197 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 2d5bbbc4-fa02-4480-b887-9b6f4980eb1b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.197 INFO edae0731-d549-4a19-a15b-e1c9f01172b4 WaitActive WaitActive WaitActive
Sep 22 23:21:38.197 INFO [1] downstairs client at 127.0.0.1:63407 has UUID 635f8034-f948-4642-a596-c4452884ff7b
Sep 22 23:21:38.197 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 635f8034-f948-4642-a596-c4452884ff7b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.197 INFO edae0731-d549-4a19-a15b-e1c9f01172b4 WaitActive WaitActive WaitActive
Sep 22 23:21:38.197 INFO [2] downstairs client at 127.0.0.1:42114 has UUID d6053127-e474-4389-afe6-5018c4a351d2
Sep 22 23:21:38.197 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d6053127-e474-4389-afe6-5018c4a351d2, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.197 INFO edae0731-d549-4a19-a15b-e1c9f01172b4 WaitActive WaitActive WaitActive
Sep 22 23:21:38.197 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.198 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:38.198 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.198 DEBG up_ds_listen was notified
Sep 22 23:21:38.198 DEBG up_ds_listen process 1002
Sep 22 23:21:38.198 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.198 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.198 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.198 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.198 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.198 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.198 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.198 DEBG up_ds_listen was notified
Sep 22 23:21:38.198 DEBG up_ds_listen process 1001
Sep 22 23:21:38.198 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.198 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:21:38.198 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.198 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.199 INFO [0] edae0731-d549-4a19-a15b-e1c9f01172b4 (d263b9b4-f13a-4d85-ac47-3dad10fd9290) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.199 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.199 WARN [0] new RM replaced this: None
Sep 22 23:21:38.199 INFO [0] Starts reconcile loop
Sep 22 23:21:38.199 INFO [1] edae0731-d549-4a19-a15b-e1c9f01172b4 (d263b9b4-f13a-4d85-ac47-3dad10fd9290) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.199 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.199 WARN [1] new RM replaced this: None
Sep 22 23:21:38.199 INFO [1] Starts reconcile loop
Sep 22 23:21:38.199 INFO [2] edae0731-d549-4a19-a15b-e1c9f01172b4 (d263b9b4-f13a-4d85-ac47-3dad10fd9290) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.199 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.199 WARN [2] new RM replaced this: None
Sep 22 23:21:38.199 INFO [2] Starts reconcile loop
Sep 22 23:21:38.199 INFO [0] 127.0.0.1:39258 task reports connection:true
Sep 22 23:21:38.199 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.199 INFO edae0731-d549-4a19-a15b-e1c9f01172b4 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.199 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.199 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.199 DEBG up_ds_listen was notified
Sep 22 23:21:38.199 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.199 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.199 DEBG up_ds_listen process 1001
Sep 22 23:21:38.199 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.199 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.199 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.199 DEBG Write     :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.199 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.199 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.199 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.199 INFO Max found gen is 1
Sep 22 23:21:38.199 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.199 INFO Next flush: 1
Sep 22 23:21:38.199 INFO All extents match
Sep 22 23:21:38.199 INFO No downstairs repair required
Sep 22 23:21:38.199 INFO No initial repair work was required
Sep 22 23:21:38.199 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.199 INFO edae0731-d549-4a19-a15b-e1c9f01172b4 is now active with session: d263b9b4-f13a-4d85-ac47-3dad10fd9290
Sep 22 23:21:38.199 INFO edae0731-d549-4a19-a15b-e1c9f01172b4 Set Active after no repair
Sep 22 23:21:38.199 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.200 INFO Set check for repair
Sep 22 23:21:38.200 DEBG Write     :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.200 INFO [1] 127.0.0.1:63407 task reports connection:true
Sep 22 23:21:38.200 INFO edae0731-d549-4a19-a15b-e1c9f01172b4 Active Active Active
Sep 22 23:21:38.200 INFO Set check for repair
Sep 22 23:21:38.200 INFO [2] 127.0.0.1:42114 task reports connection:true
Sep 22 23:21:38.200 INFO edae0731-d549-4a19-a15b-e1c9f01172b4 Active Active Active
Sep 22 23:21:38.200 INFO Set check for repair
Sep 22 23:21:38.200 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.200 INFO [0] received reconcile message
Sep 22 23:21:38.200 DEBG Write     :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.200 INFO [0] All repairs completed, exit
Sep 22 23:21:38.200 INFO [0] Starts cmd_loop
Sep 22 23:21:38.200 INFO [1] received reconcile message
Sep 22 23:21:38.200 INFO [1] All repairs completed, exit
Sep 22 23:21:38.200 INFO [1] Starts cmd_loop
Sep 22 23:21:38.200 INFO [2] received reconcile message
Sep 22 23:21:38.200 INFO [2] All repairs completed, exit
Sep 22 23:21:38.200 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.201 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.203 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:38.203 DEBG IO Write 1001 has deps [JobId(1000)]
Sep 22 23:21:38.203 DEBG up_ds_listen was notified
Sep 22 23:21:38.203 DEBG up_ds_listen process 1000
Sep 22 23:21:38.203 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.203 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.203 DEBG up_ds_listen was notified
Sep 22 23:21:38.203 DEBG up_ds_listen process 1001
Sep 22 23:21:38.203 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.203 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.204 DEBG IO Write 1000 has deps []
Sep 22 23:21:38.206 DEBG up_ds_listen was notified
Sep 22 23:21:38.206 DEBG up_ds_listen process 1000
Sep 22 23:21:38.206 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.206 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.206 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:21:38.206 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.206 DEBG up_ds_listen was notified
Sep 22 23:21:38.207 DEBG up_ds_listen process 1002
Sep 22 23:21:38.207 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.207 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.207 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.207 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.207 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.207 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.207 INFO Created new region file "/tmp/downstairs-XotxiwRu/region.json"
Sep 22 23:21:38.207 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.208 DEBG Write     :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.208 DEBG IO Read  1003 has deps [JobId(1002), JobId(1001), JobId(1000)]
Sep 22 23:21:38.208 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:38.208 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.209 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.209 DEBG up_ds_listen was notified
Sep 22 23:21:38.209 DEBG up_ds_listen process 1001
Sep 22 23:21:38.209 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.209 DEBG Read      :1003 deps:[JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.209 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.209 DEBG IO Read  1001 has deps [JobId(1000)]
Sep 22 23:21:38.210 DEBG Read      :1003 deps:[JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.210 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:38.210 DEBG Read      :1003 deps:[JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.210 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.210 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.210 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:38.211 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:38.212 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.212 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:38.212 DEBG up_ds_listen was notified
Sep 22 23:21:38.212 DEBG up_ds_listen process 1001
Sep 22 23:21:38.212 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.212 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.213 DEBG up_ds_listen was notified
Sep 22 23:21:38.213 DEBG up_ds_listen process 1002
Sep 22 23:21:38.213 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.213 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.213 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
Sep 22 23:21:38.214 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.215 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:21:38.215 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.215 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:38.215 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.218 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.220 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:21:38.220 DEBG up_ds_listen was notified
Sep 22 23:21:38.220 DEBG up_ds_listen process 1002
Sep 22 23:21:38.220 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.220 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.221 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.221 DEBG up_ds_listen was notified
Sep 22 23:21:38.221 DEBG up_ds_listen process 1003
Sep 22 23:21:38.221 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:38.221 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.222 DEBG IO Read  1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:38.222 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.223 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.223 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.223 DEBG IO Write 1002 has deps [JobId(1001), JobId(1000)]
Sep 22 23:21:38.223 DEBG Read      :1002 deps:[JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.226 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.226 DEBG up_ds_listen was notified
Sep 22 23:21:38.226 DEBG up_ds_listen process 1003
Sep 22 23:21:38.226 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:38.226 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.227 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
Sep 22 23:21:38.228 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.228 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.228 DEBG [0] Read AckReady 1002, : downstairs
Sep 22 23:21:38.228 DEBG up_ds_listen was notified
Sep 22 23:21:38.228 DEBG up_ds_listen process 1002
Sep 22 23:21:38.228 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.228 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.229 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.229 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
Sep 22 23:21:38.230 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.230 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.231 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.231 DEBG [1] Read already AckReady 1002, : downstairs
Sep 22 23:21:38.233 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:21:38.234 DEBG [2] Read already AckReady 1002, : downstairs
Sep 22 23:21:38.234 DEBG up_ds_listen was notified
Sep 22 23:21:38.234 DEBG up_ds_listen process 1002
Sep 22 23:21:38.234 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:38.235 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.235 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:21:38.236 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.238 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.239 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.239 DEBG up_ds_listen was notified
Sep 22 23:21:38.239 DEBG up_ds_listen process 1003
Sep 22 23:21:38.239 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:38.239 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.240 DEBG [2] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.240 DEBG up_ds_listen was notified
Sep 22 23:21:38.240 DEBG up_ds_listen process 1003
Sep 22 23:21:38.240 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:38.240 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.240 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.240 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.241 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.241 DEBG IO Read  1003 has deps [JobId(1002), JobId(1000)]
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.241326829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:51585","component":"dropshot","remote_addr":"127.0.0.1:63359"}
Sep 22 23:21:38.242 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.243 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
Sep 22 23:21:38.243 DEBG Read      :1003 deps:[JobId(1002), JobId(1000)] res:true
test test::integration_test_volume_write_unwritten_subvols_3 ... ok
{"msg":"no entry exists for volume 16548ffc-77b2-48b7-a37b-cfc57a4b3680, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.243747539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.243 INFO current number of open files limit 65536 is already the maximum
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.243958755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244018823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244052619Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.244 INFO Created new region file "/tmp/downstairs-sxN8iMqh/region.json"
{"msg":"Crucible stats registered with UUID: d17b320f-b32d-4e32-93ab-fda757545dc7","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244078099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible d17b320f-b32d-4e32-93ab-fda757545dc7 has session id: bead0ee0-d667-4905-9676-91a0e4e103e0","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244103002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:39369","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244235544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:55268","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244482105Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:37306","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244549568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244629766Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244674709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244711652Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 16548ffc-77b2-48b7-a37b-cfc57a4b3680 constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.244939336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"d17b320f-b32d-4e32-93ab-fda757545dc7 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.245187295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] d17b320f-b32d-4e32-93ab-fda757545dc7 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.245374783Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:39369 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.245408145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] d17b320f-b32d-4e32-93ab-fda757545dc7 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.245433706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:55268 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.245457674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] d17b320f-b32d-4e32-93ab-fda757545dc7 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.245480479Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:37306 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.245509212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.245 INFO accepted connection from 127.0.0.1:39935, task: main
Sep 22 23:21:38.245 INFO accepted connection from 127.0.0.1:35109, task: main
Sep 22 23:21:38.245 INFO accepted connection from 127.0.0.1:34869, task: main
Sep 22 23:21:38.246 INFO Connection request from d17b320f-b32d-4e32-93ab-fda757545dc7 with version 4, task: proc
Sep 22 23:21:38.246 INFO upstairs UpstairsConnection { upstairs_id: d17b320f-b32d-4e32-93ab-fda757545dc7, session_id: c6dc54f3-5d65-4987-826a-9776dd72d00d, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.246 INFO Connection request from d17b320f-b32d-4e32-93ab-fda757545dc7 with version 4, task: proc
Sep 22 23:21:38.246 INFO upstairs UpstairsConnection { upstairs_id: d17b320f-b32d-4e32-93ab-fda757545dc7, session_id: c6dc54f3-5d65-4987-826a-9776dd72d00d, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.246 INFO Connection request from d17b320f-b32d-4e32-93ab-fda757545dc7 with version 4, task: proc
Sep 22 23:21:38.246 INFO upstairs UpstairsConnection { upstairs_id: d17b320f-b32d-4e32-93ab-fda757545dc7, session_id: c6dc54f3-5d65-4987-826a-9776dd72d00d, gen: 1 } connected, version 4, task: proc
{"msg":"[0] d17b320f-b32d-4e32-93ab-fda757545dc7 (c6dc54f3-5d65-4987-826a-9776dd72d00d) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.246551025Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.246602534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session c6dc54f3-5d65-4987-826a-9776dd72d00d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.246644499Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] d17b320f-b32d-4e32-93ab-fda757545dc7 (c6dc54f3-5d65-4987-826a-9776dd72d00d) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.246690607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.246730838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session c6dc54f3-5d65-4987-826a-9776dd72d00d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.246771851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.246 INFO UpstairsConnection { upstairs_id: d17b320f-b32d-4e32-93ab-fda757545dc7, session_id: c6dc54f3-5d65-4987-826a-9776dd72d00d, gen: 1 } is now active (read-write)
{"msg":"[2] d17b320f-b32d-4e32-93ab-fda757545dc7 (c6dc54f3-5d65-4987-826a-9776dd72d00d) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.246810941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.246845545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session c6dc54f3-5d65-4987-826a-9776dd72d00d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.246882298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.246 INFO UpstairsConnection { upstairs_id: d17b320f-b32d-4e32-93ab-fda757545dc7, session_id: c6dc54f3-5d65-4987-826a-9776dd72d00d, gen: 1 } is now active (read-write)
Sep 22 23:21:38.247 INFO UpstairsConnection { upstairs_id: d17b320f-b32d-4e32-93ab-fda757545dc7, session_id: c6dc54f3-5d65-4987-826a-9776dd72d00d, gen: 1 } is now active (read-write)
test test::integration_test_volume_write_unwritten_subvols ... ok
Sep 22 23:21:38.247 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.247 DEBG [0] Read AckReady 1003, : downstairs
Sep 22 23:21:38.247 INFO Created new region file "/tmp/downstairs-lRuGJNxo/region.json"
{"msg":"[0] downstairs client at 127.0.0.1:39369 has UUID 07ed70f5-1d11-4ea6-9c11-5ff16ec9fcb3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.247630436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 07ed70f5-1d11-4ea6-9c11-5ff16ec9fcb3, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.247691118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"d17b320f-b32d-4e32-93ab-fda757545dc7 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.247734802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:55268 has UUID fbc1ff7a-5e2f-4a0a-a554-498b43e005a3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.24777586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: fbc1ff7a-5e2f-4a0a-a554-498b43e005a3, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.247823457Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"d17b320f-b32d-4e32-93ab-fda757545dc7 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.247866102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:37306 has UUID 89e24c32-e4e6-442a-a2a4-07d6620fd2c7","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.247908929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 89e24c32-e4e6-442a-a2a4-07d6620fd2c7, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.247960099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"d17b320f-b32d-4e32-93ab-fda757545dc7 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.248003049Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.248 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.248 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.248 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.248 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.249 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.249 DEBG [1] Read already AckReady 1003, : downstairs
Sep 22 23:21:38.249 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] d17b320f-b32d-4e32-93ab-fda757545dc7 (c6dc54f3-5d65-4987-826a-9776dd72d00d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250059456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250120847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.250163829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250204913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] d17b320f-b32d-4e32-93ab-fda757545dc7 (c6dc54f3-5d65-4987-826a-9776dd72d00d) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250250143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250289452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.250327124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250368131Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] d17b320f-b32d-4e32-93ab-fda757545dc7 (c6dc54f3-5d65-4987-826a-9776dd72d00d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250414266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.250 INFO current number of open files limit 65536 is already the maximum
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250457829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.250 INFO Opened existing region file "/tmp/downstairs-lRuGJNxo/region.json"
Sep 22 23:21:38.250 INFO Database read version 1
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.250535181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250612402Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:39369 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250656855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.250 INFO Database write version 1
{"msg":"d17b320f-b32d-4e32-93ab-fda757545dc7 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250693174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250730317Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250770218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250809221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250849516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250891269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250929445Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.250968738Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251005816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251042057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251081141Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251122268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.2511606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251199762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.251 DEBG [2] Read already AckReady 1003, : downstairs
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251241009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251279812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.251 DEBG up_ds_listen was notified
Sep 22 23:21:38.251 DEBG up_ds_listen process 1003
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251307788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.251 DEBG [A] ack job 1003:4, : downstairs
{"msg":"d17b320f-b32d-4e32-93ab-fda757545dc7 is now active with session: c6dc54f3-5d65-4987-826a-9776dd72d00d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251347418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"d17b320f-b32d-4e32-93ab-fda757545dc7 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.25139029Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251429391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251464564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:55268 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251499659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"d17b320f-b32d-4e32-93ab-fda757545dc7 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251543856Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251583475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:37306 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251622023Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.251 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"d17b320f-b32d-4e32-93ab-fda757545dc7 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251660962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.25169786Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251738604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.251 INFO UUID: c25278bf-5b9e-4380-abf6-dfd9701dce27
Sep 22 23:21:38.251 INFO Blocks per extent:5 Total Extents: 2
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.25177731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251895485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.251 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.251 INFO Upstairs <-> Downstairs Message Version: 4, task: main
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251941058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.251 INFO Using address: 127.0.0.1:33527, task: main
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.251979177Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.252021253Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.252061047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.252098126Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.252136609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 16548ffc-77b2-48b7-a37b-cfc57a4b3680 activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.25217731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 16548ffc-77b2-48b7-a37b-cfc57a4b3680 constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.252218268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.252259918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680","method":"POST","req_id":"be303c4e-f00d-4251-8e01-091f9f93f1f2","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":9256,"response_code":"200"}
Sep 22 23:21:38.252 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.252 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58793, task: repair
Sep 22 23:21:38.252 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58793, task: repair
Sep 22 23:21:38.252 INFO listening, local_addr: 127.0.0.1:58793, task: repair
Sep 22 23:21:38.252 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58793, task: repair
Sep 22 23:21:38.252 INFO Using repair address: 127.0.0.1:58793, task: main
Sep 22 23:21:38.252 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.253 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.253 INFO Created new region file "/tmp/downstairs-ZkSPwZxy/region.json"
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.253828028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_write","method":"POST","req_id":"56b4dd93-c533-4f0b-8c7b-8238348be4a1","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":1587,"response_code":"204"}
Sep 22 23:21:38.255 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.255 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.256 DEBG Write     :1000 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.258666493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_write","method":"POST","req_id":"cd39d46d-e128-42ea-8288-99336e3a80ef","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":1360,"response_code":"204"}
Sep 22 23:21:38.258 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.258 INFO Opened existing region file "/tmp/downstairs-ZkSPwZxy/region.json"
Sep 22 23:21:38.258 INFO Database read version 1
Sep 22 23:21:38.259 INFO Database write version 1
Sep 22 23:21:38.259 DEBG Write     :1001 deps:[] res:true
test test::integration_test_volume_write_unwritten_subvols_sparse ... ok
Sep 22 23:21:38.259 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:38.259 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.259 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:38.259 INFO Created new region file "/tmp/downstairs-9KQd6X6V/region.json"
Sep 22 23:21:38.261 INFO UUID: 99808fa8-ed0d-4880-abf3-c77623f8cd97
Sep 22 23:21:38.261 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.261 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.261 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.261 INFO Using address: 127.0.0.1:59188, task: main
Sep 22 23:21:38.261 INFO Repair listens on 127.0.0.1:0, task: repair
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.261680267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_write","method":"POST","req_id":"451ed998-29e4-43cb-9662-7473e4fde8da","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":1323,"response_code":"204"}
Sep 22 23:21:38.261 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:33494, task: repair
Sep 22 23:21:38.261 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:33494, task: repair
Sep 22 23:21:38.261 INFO listening, local_addr: 127.0.0.1:33494, task: repair
Sep 22 23:21:38.261 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:33494, task: repair
Sep 22 23:21:38.262 INFO Using repair address: 127.0.0.1:33494, task: main
Sep 22 23:21:38.262 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.262 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.262 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.262 INFO Created new region file "/tmp/downstairs-lX8N7uaO/region.json"
Sep 22 23:21:38.263 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.263 DEBG Write     :1002 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.265616183Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_write","method":"POST","req_id":"7855f821-41fa-4f86-a042-757c2b9ce7b4","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":1449,"response_code":"204"}
Sep 22 23:21:38.266 DEBG Write     :1003 deps:[] res:true
Sep 22 23:21:38.266 DEBG Write     :1003 deps:[] res:true
Sep 22 23:21:38.266 DEBG Write     :1003 deps:[] res:true
Sep 22 23:21:38.268 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.268 INFO Opened existing region file "/tmp/downstairs-lX8N7uaO/region.json"
Sep 22 23:21:38.268 INFO Database read version 1
Sep 22 23:21:38.268 INFO Database write version 1
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.268931276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_write","method":"POST","req_id":"3b06b483-774a-4581-b001-3f530562d342","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":1153,"response_code":"204"}
Sep 22 23:21:38.269 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:38.269 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:38.270 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:38.272 INFO UUID: 651c2c7a-b2ed-48c3-b57d-44ab91a6f440
Sep 22 23:21:38.272 INFO Blocks per extent:5 Total Extents: 2
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.272136874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_write","method":"POST","req_id":"b2d950b3-9eee-4cba-80c0-bff75bd19d02","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":1341,"response_code":"204"}
Sep 22 23:21:38.272 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.272 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.272 INFO Using address: 127.0.0.1:54900, task: main
Sep 22 23:21:38.272 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.272 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35114, task: repair
Sep 22 23:21:38.272 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35114, task: repair
Sep 22 23:21:38.272 INFO listening, local_addr: 127.0.0.1:35114, task: repair
Sep 22 23:21:38.272 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35114, task: repair
Sep 22 23:21:38.272 INFO Using repair address: 127.0.0.1:35114, task: main
Sep 22 23:21:38.272 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.273 DEBG Write     :1005 deps:[] res:true
note: configured to log to "/dev/stdout"
Sep 22 23:21:38.273 DEBG Write     :1005 deps:[] res:true
Sep 22 23:21:38.274 DEBG Write     :1005 deps:[] res:true
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.27460174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:55883","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:55883","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.27478589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.276593127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_write","method":"POST","req_id":"63600948-3668-44fc-8dc9-d0c64f1a5c93","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":1451,"response_code":"204"}
Sep 22 23:21:38.277 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:38.277 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:38.277 DEBG Write     :1006 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.280091771Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_write","method":"POST","req_id":"be829a86-e982-4672-8966-74ca3433b266","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":1436,"response_code":"204"}
Sep 22 23:21:38.280 DEBG Write     :1007 deps:[] res:true
Sep 22 23:21:38.280 DEBG Write     :1007 deps:[] res:true
Sep 22 23:21:38.281 DEBG Write     :1007 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.283472942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_write","method":"POST","req_id":"3d4340b2-0490-4162-a3ff-806301c41c2b","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":1417,"response_code":"204"}
Sep 22 23:21:38.284 DEBG Write     :1008 deps:[] res:true
Sep 22 23:21:38.284 DEBG Write     :1008 deps:[] res:true
Sep 22 23:21:38.284 DEBG Write     :1008 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.286993336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_write","method":"POST","req_id":"a1ea2512-259c-446d-8e01-3bb394cc5bb6","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":1375,"response_code":"204"}
Sep 22 23:21:38.287 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:38.288 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:38.288 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:38.290 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:21:38.291 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:21:38.291 DEBG Read      :1010 deps:[JobId(1000)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.293334928Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"b5d5c091-ed3a-47ee-b6c8-ded69fbb01c1","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":3911,"response_code":"200"}
Sep 22 23:21:38.294 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:21:38.295 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:21:38.295 DEBG Read      :1011 deps:[JobId(1001)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.29661142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"52c2828e-9b2a-487b-96ce-4ded00679350","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":2562,"response_code":"200"}
Sep 22 23:21:38.298 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:21:38.298 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:21:38.298 DEBG Read      :1012 deps:[JobId(1002)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.299726613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"0b94ac49-1f6c-4b0e-aa0c-67cef59b9be8","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":2530,"response_code":"200"}
Sep 22 23:21:38.301 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:21:38.301 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:21:38.302 DEBG Read      :1013 deps:[JobId(1003)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.303958173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"a87c80a2-e124-422a-955a-81caa2c37315","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":3511,"response_code":"200"}
Sep 22 23:21:38.305 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:21:38.306 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:21:38.306 DEBG Read      :1014 deps:[JobId(1004)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.308077063Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"269ff15b-d9c8-451c-915a-4fd1718d5b5a","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":3400,"response_code":"200"}
Sep 22 23:21:38.310 DEBG Read      :1015 deps:[JobId(1005)] res:true
Sep 22 23:21:38.310 DEBG Read      :1015 deps:[JobId(1005)] res:true
Sep 22 23:21:38.310 DEBG Read      :1015 deps:[JobId(1005)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.312423468Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"aa7a1fc7-c999-4c03-b17e-4d27a531a8dd","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":3600,"response_code":"200"}
Sep 22 23:21:38.314 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:21:38.314 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:21:38.314 DEBG Read      :1016 deps:[JobId(1006)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.316365716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"62a9d1af-c42a-42d4-bf8a-80d715759cd9","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":3260,"response_code":"200"}
Sep 22 23:21:38.318 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:21:38.318 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:21:38.318 DEBG Read      :1017 deps:[JobId(1007)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.319919963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"0c974305-b9bc-4b44-a358-65991b73ac64","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":2854,"response_code":"200"}
Sep 22 23:21:38.321 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:21:38.321 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:21:38.321 DEBG Read      :1018 deps:[JobId(1008)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.323020181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"b4b3215a-bb8a-4800-abac-5c796f2b91bb","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":2524,"response_code":"200"}
Sep 22 23:21:38.324 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:21:38.324 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:21:38.324 DEBG Read      :1019 deps:[JobId(1009)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.326234252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"73e0f1b6-e16d-4263-883f-2a541ea6960e","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":2609,"response_code":"200"}
Sep 22 23:21:38.328 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.328 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:21:38.329 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.338012959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680/bulk_read","method":"POST","req_id":"8905ea72-8884-4a07-9fa7-7341d81decfa","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":11073,"response_code":"200"}
{"msg":"detach removing entry for volume 16548ffc-77b2-48b7-a37b-cfc57a4b3680","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.338826565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume 16548ffc-77b2-48b7-a37b-cfc57a4b3680","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.338886146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.340 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.340 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.340 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.341546433Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"d17b320f-b32d-4e32-93ab-fda757545dc7 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.341597178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.341754989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/16548ffc-77b2-48b7-a37b-cfc57a4b3680","method":"DELETE","req_id":"eaec3b2c-ced2-4377-aebc-b28f63a247a2","remote_addr":"127.0.0.1:63359","local_addr":"127.0.0.1:51585","component":"dropshot","latency_us":3063,"response_code":"204"}
test test::test_pantry_bulk_read ... ok
Sep 22 23:21:38.350 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.350 INFO Created new region file "/tmp/downstairs-BqJQGXV6/region.json"
Sep 22 23:21:38.355 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.356 INFO Opened existing region file "/tmp/downstairs-BqJQGXV6/region.json"
Sep 22 23:21:38.356 INFO Database read version 1
Sep 22 23:21:38.356 INFO Database write version 1
Sep 22 23:21:38.359 INFO UUID: ad667607-b05e-4427-9709-433732d1d5ab
Sep 22 23:21:38.359 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.359 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.359 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.359 INFO Using address: 127.0.0.1:65363, task: main
Sep 22 23:21:38.359 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.359 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:57934, task: repair
Sep 22 23:21:38.359 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:57934, task: repair
Sep 22 23:21:38.359 INFO listening, local_addr: 127.0.0.1:57934, task: repair
Sep 22 23:21:38.360 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:57934, task: repair
Sep 22 23:21:38.360 INFO Using repair address: 127.0.0.1:57934, task: main
Sep 22 23:21:38.360 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.360 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.360 INFO Created new region file "/tmp/downstairs-BpCJ2qOn/region.json"
Sep 22 23:21:38.366 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.367 INFO Opened existing region file "/tmp/downstairs-BpCJ2qOn/region.json"
Sep 22 23:21:38.367 INFO Database read version 1
Sep 22 23:21:38.367 INFO Database write version 1
Sep 22 23:21:38.369 INFO UUID: 995b5b97-67dd-43f0-b20a-d5b2c0c00924
Sep 22 23:21:38.369 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:38.369 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.369 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.369 INFO Using address: 127.0.0.1:63095, task: main
Sep 22 23:21:38.369 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.370 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37590, task: repair
Sep 22 23:21:38.370 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37590, task: repair
Sep 22 23:21:38.370 INFO listening, local_addr: 127.0.0.1:37590, task: repair
Sep 22 23:21:38.370 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37590, task: repair
Sep 22 23:21:38.370 INFO Using repair address: 127.0.0.1:37590, task: main
Sep 22 23:21:38.370 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.370 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.370 INFO Created new region file "/tmp/downstairs-iM5wyDOR/region.json"
Sep 22 23:21:38.376 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.376 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.376 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.376 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.376 INFO Opened existing region file "/tmp/downstairs-iM5wyDOR/region.json"
Sep 22 23:21:38.376 INFO Database read version 1
Sep 22 23:21:38.376 INFO Database write version 1
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.376904639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:55883","component":"dropshot","remote_addr":"127.0.0.1:43786"}
{"msg":"no entry exists for volume 17fca985-4bd4-4289-96a4-634e1d483fcb, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.377701347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.377872543Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.377935634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.37796387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: e5b4ada7-7239-4d69-8861-23d72de6320f","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.377991869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible e5b4ada7-7239-4d69-8861-23d72de6320f has session id: 53f0eb62-df50-4075-93bc-d1b8847e7671","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.378034524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:33527","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.378075992Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:59188","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.378229491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:54900","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.378366465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.378416398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.378455981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.378486116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 17fca985-4bd4-4289-96a4-634e1d483fcb constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.378615058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.378835689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.378 INFO accepted connection from 127.0.0.1:55430, task: main
Sep 22 23:21:38.379 INFO accepted connection from 127.0.0.1:50291, task: main
Sep 22 23:21:38.379 INFO accepted connection from 127.0.0.1:55513, task: main
{"msg":"[1] e5b4ada7-7239-4d69-8861-23d72de6320f looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.37928768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:59188 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.379338906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] e5b4ada7-7239-4d69-8861-23d72de6320f looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.379378071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:54900 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.379417869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] e5b4ada7-7239-4d69-8861-23d72de6320f looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.379457955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:33527 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.379501723Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.379 INFO Connection request from e5b4ada7-7239-4d69-8861-23d72de6320f with version 4, task: proc
Sep 22 23:21:38.379 INFO upstairs UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.379 INFO Connection request from e5b4ada7-7239-4d69-8861-23d72de6320f with version 4, task: proc
Sep 22 23:21:38.379 INFO upstairs UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.379 INFO Connection request from e5b4ada7-7239-4d69-8861-23d72de6320f with version 4, task: proc
Sep 22 23:21:38.379 INFO upstairs UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } connected, version 4, task: proc
{"msg":"[1] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.379975491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.380032681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 22467938-6385-426f-86c0-fa56c94d98b1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.380083528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.380142259Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.380190019Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 22467938-6385-426f-86c0-fa56c94d98b1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.380239267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.380288491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.380 INFO UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } is now active (read-write)
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.380329015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.380 INFO UUID: f0f4ef77-64c9-4cae-ad93-ac2fb4943d50
Sep 22 23:21:38.380 INFO Blocks per extent:5 Total Extents: 2
{"msg":"[0] client is_active_req TRUE, promote! session 22467938-6385-426f-86c0-fa56c94d98b1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.380374879Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.380 INFO UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } is now active (read-write)
Sep 22 23:21:38.380 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.380 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.380 INFO Using address: 127.0.0.1:64907, task: main
Sep 22 23:21:38.380 INFO UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } is now active (read-write)
Sep 22 23:21:38.380 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.380 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:34369, task: repair
Sep 22 23:21:38.380 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:34369, task: repair
Sep 22 23:21:38.380 INFO listening, local_addr: 127.0.0.1:34369, task: repair
{"msg":"[1] downstairs client at 127.0.0.1:59188 has UUID 99808fa8-ed0d-4880-abf3-c77623f8cd97","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.381012291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.381 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:34369, task: repair
Sep 22 23:21:38.381 INFO Using repair address: 127.0.0.1:34369, task: main
Sep 22 23:21:38.381 INFO No SSL acceptor configured, task: main
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 99808fa8-ed0d-4880-abf3-c77623f8cd97, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.381075392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.381132582Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:54900 has UUID 651c2c7a-b2ed-48c3-b57d-44ab91a6f440","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.381183118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 651c2c7a-b2ed-48c3-b57d-44ab91a6f440, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.381235648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.381274656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] downstairs client at 127.0.0.1:33527 has UUID c25278bf-5b9e-4380-abf6-dfd9701dce27","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.381318668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c25278bf-5b9e-4380-abf6-dfd9701dce27, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.381371542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.381418903Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.381 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.381 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.381 INFO Upstairs starts
Sep 22 23:21:38.381 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.381 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.381 INFO Crucible stats registered with UUID: f5cf38e1-07a4-4403-b783-c67b3e5d3b37
Sep 22 23:21:38.381 INFO Crucible f5cf38e1-07a4-4403-b783-c67b3e5d3b37 has session id: 0e0eeacf-f1e3-4f53-b5ee-c5a33ce2f2b6
Sep 22 23:21:38.381 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.381 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.381 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.382 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.382 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.382 INFO [0] connecting to 127.0.0.1:65363, looper: 0
Sep 22 23:21:38.382 INFO [1] connecting to 127.0.0.1:63095, looper: 1
Sep 22 23:21:38.382 INFO [2] connecting to 127.0.0.1:64907, looper: 2
Sep 22 23:21:38.382 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.382 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.382 INFO Flush timeout: 0.5
Sep 22 23:21:38.382 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.382 INFO accepted connection from 127.0.0.1:46576, task: main
Sep 22 23:21:38.382 INFO accepted connection from 127.0.0.1:61396, task: main
Sep 22 23:21:38.382 INFO accepted connection from 127.0.0.1:56109, task: main
Sep 22 23:21:38.382 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.382 INFO [0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 looper connected, looper: 0
Sep 22 23:21:38.382 INFO [0] Proc runs for 127.0.0.1:65363 in state New
Sep 22 23:21:38.382 INFO [1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 looper connected, looper: 1
Sep 22 23:21:38.382 INFO [1] Proc runs for 127.0.0.1:63095 in state New
Sep 22 23:21:38.382 INFO [2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 looper connected, looper: 2
Sep 22 23:21:38.382 INFO [2] Proc runs for 127.0.0.1:64907 in state New
{"msg":"[1] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.382938988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.38297976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.383009223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383049379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383096014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383139635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.383 INFO Connection request from f5cf38e1-07a4-4403-b783-c67b3e5d3b37 with version 4, task: proc
Sep 22 23:21:38.383 INFO upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } connected, version 4, task: proc
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.383180655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383221055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
Sep 22 23:21:38.383 INFO Connection request from f5cf38e1-07a4-4403-b783-c67b3e5d3b37 with version 4, task: proc
Sep 22 23:21:38.383 INFO upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } connected, version 4, task: proc
{"msg":"[0] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383270459Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383313642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.383353484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.383 INFO Connection request from f5cf38e1-07a4-4403-b783-c67b3e5d3b37 with version 4, task: proc
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.3833929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.383 INFO upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } connected, version 4, task: proc
{"msg":"[1] 127.0.0.1:59188 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.38343736Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383478656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.38351252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383536036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383565758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383604728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.383 INFO [0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (02fbce88-f31f-4b2e-a8ed-d7530615b97b) New New New ds_transition to WaitActive
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383641722Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.383 INFO [0] Transition from New to WaitActive
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383677759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383716528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.383 INFO [1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (02fbce88-f31f-4b2e-a8ed-d7530615b97b) WaitActive New New ds_transition to WaitActive
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383753724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.383 INFO [1] Transition from New to WaitActive
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383779703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383808841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.383 INFO [2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (02fbce88-f31f-4b2e-a8ed-d7530615b97b) WaitActive WaitActive New ds_transition to WaitActive
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383848008Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.383 INFO [2] Transition from New to WaitActive
Sep 22 23:21:38.383 INFO UUID: 63b48fd3-ffb2-4c20-bff7-4a64980fdfc8
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383886954Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.383 INFO Blocks per extent:512 Total Extents: 188
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383922694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.383963264Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
Sep 22 23:21:38.383 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384001774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.384 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.384 INFO Using address: 127.0.0.1:52165, task: main
Sep 22 23:21:38.384 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 active request set
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384041969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f is now active with session: 22467938-6385-426f-86c0-fa56c94d98b1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384084772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384121675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.384 INFO [0] received activate with gen 1
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.38414818Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.384 INFO [0] client got ds_active_rx, promote! session 02fbce88-f31f-4b2e-a8ed-d7530615b97b
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384170829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:54900 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384193463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384228347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.384 INFO [1] received activate with gen 1
Sep 22 23:21:38.384 INFO [1] client got ds_active_rx, promote! session 02fbce88-f31f-4b2e-a8ed-d7530615b97b
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384263567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:33527 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384291534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384320089Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.384 INFO [2] received activate with gen 1
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384351248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.384 INFO [2] client got ds_active_rx, promote! session 02fbce88-f31f-4b2e-a8ed-d7530615b97b
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384382056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384418268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384442834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384464707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384490545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.384 INFO UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } is now active (read-write)
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384528558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384568189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384603488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.384 INFO UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } is now active (read-write)
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384626686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 17fca985-4bd4-4289-96a4-634e1d483fcb activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384649727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 17fca985-4bd4-4289-96a4-634e1d483fcb constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384672667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.384 INFO Repair listens on 127.0.0.1:0, task: repair
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.384694153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb","method":"POST","req_id":"d8af2e67-68cb-43e5-8478-69f7b6054043","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":6298,"response_code":"200"}
Sep 22 23:21:38.384 INFO UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } is now active (read-write)
Sep 22 23:21:38.384 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:50383, task: repair
Sep 22 23:21:38.384 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:50383, task: repair
Sep 22 23:21:38.384 INFO listening, local_addr: 127.0.0.1:50383, task: repair
Sep 22 23:21:38.385 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:50383, task: repair
Sep 22 23:21:38.385 INFO Using repair address: 127.0.0.1:50383, task: main
Sep 22 23:21:38.385 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.385 INFO [0] downstairs client at 127.0.0.1:65363 has UUID ad667607-b05e-4427-9709-433732d1d5ab
Sep 22 23:21:38.385 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ad667607-b05e-4427-9709-433732d1d5ab, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.385 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitActive WaitActive WaitActive
Sep 22 23:21:38.385 INFO [1] downstairs client at 127.0.0.1:63095 has UUID 995b5b97-67dd-43f0-b20a-d5b2c0c00924
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.385364051Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb/bulk_write","method":"POST","req_id":"3ba8385d-17a3-4c27-ab54-6face2758159","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":1059,"response_code":"204"}
Sep 22 23:21:38.385 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 995b5b97-67dd-43f0-b20a-d5b2c0c00924, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.385 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitActive WaitActive WaitActive
Sep 22 23:21:38.385 INFO [2] downstairs client at 127.0.0.1:64907 has UUID f0f4ef77-64c9-4cae-ad93-ac2fb4943d50
Sep 22 23:21:38.385 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f0f4ef77-64c9-4cae-ad93-ac2fb4943d50, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.385 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitActive WaitActive WaitActive
Sep 22 23:21:38.385 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.386 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.386 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.386 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.386 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.386 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.386 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.386 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.387 INFO [0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (02fbce88-f31f-4b2e-a8ed-d7530615b97b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.387 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.387 WARN [0] new RM replaced this: None
Sep 22 23:21:38.387 INFO [0] Starts reconcile loop
Sep 22 23:21:38.387 INFO [1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (02fbce88-f31f-4b2e-a8ed-d7530615b97b) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.387 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.387 WARN [1] new RM replaced this: None
Sep 22 23:21:38.387 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.387 INFO [1] Starts reconcile loop
Sep 22 23:21:38.387 INFO [2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (02fbce88-f31f-4b2e-a8ed-d7530615b97b) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.387 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.387 WARN [2] new RM replaced this: None
Sep 22 23:21:38.387 INFO [2] Starts reconcile loop
Sep 22 23:21:38.387 INFO [0] 127.0.0.1:65363 task reports connection:true
Sep 22 23:21:38.387 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.387 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:38.387 INFO [0]R  generation: [0, 0]
Sep 22 23:21:38.387 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.387 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:38.387 INFO [1]R  generation: [0, 0]
Sep 22 23:21:38.387 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.387 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:38.387 INFO [2]R  generation: [0, 0]
Sep 22 23:21:38.387 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.387 INFO Max found gen is 1
Sep 22 23:21:38.387 INFO Generation requested: 1 >= found:1
Sep 22 23:21:38.387 INFO Next flush: 1
Sep 22 23:21:38.387 INFO All extents match
Sep 22 23:21:38.387 INFO No downstairs repair required
Sep 22 23:21:38.387 INFO No initial repair work was required
Sep 22 23:21:38.387 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.387 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 is now active with session: 02fbce88-f31f-4b2e-a8ed-d7530615b97b
Sep 22 23:21:38.387 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Set Active after no repair
Sep 22 23:21:38.387 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.387 INFO Set check for repair
Sep 22 23:21:38.387 INFO [1] 127.0.0.1:63095 task reports connection:true
Sep 22 23:21:38.387 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Active Active Active
Sep 22 23:21:38.387 INFO Set check for repair
Sep 22 23:21:38.387 INFO [2] 127.0.0.1:64907 task reports connection:true
Sep 22 23:21:38.387 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Active Active Active
Sep 22 23:21:38.387 INFO Set check for repair
Sep 22 23:21:38.387 INFO [0] received reconcile message
Sep 22 23:21:38.387 INFO [0] All repairs completed, exit
Sep 22 23:21:38.387 INFO [0] Starts cmd_loop
Sep 22 23:21:38.387 INFO [1] received reconcile message
Sep 22 23:21:38.387 INFO [1] All repairs completed, exit
Sep 22 23:21:38.387 INFO [1] Starts cmd_loop
Sep 22 23:21:38.387 INFO [2] received reconcile message
Sep 22 23:21:38.387 INFO [2] All repairs completed, exit
Sep 22 23:21:38.387 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.388 DEBG IO Read  1000 has deps []
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.388904744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb/bulk_write","method":"POST","req_id":"4e784489-d034-4935-b724-d9e27228f651","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":1031,"response_code":"204"}
Sep 22 23:21:38.389 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.389 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:38.389 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.389 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:38.390 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.390 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:38.391 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:38.391 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:38.391 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:38.391 DEBG up_ds_listen was notified
Sep 22 23:21:38.391 DEBG up_ds_listen process 1000
Sep 22 23:21:38.391 DEBG [A] ack job 1000:1, : downstairs
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.391943944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb/bulk_write","method":"POST","req_id":"a3ee8070-9551-4a61-8484-5c0ff5f1e16c","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":1033,"response_code":"204"}
Sep 22 23:21:38.392 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.392 INFO Request to deactivate this guest
Sep 22 23:21:38.392 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 set deactivating.
Sep 22 23:21:38.392 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:21:38.392 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.392 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.392 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.392 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.393 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.393 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:38.393 INFO [0] check deactivate YES
Sep 22 23:21:38.393 INFO [0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (02fbce88-f31f-4b2e-a8ed-d7530615b97b) Active Active Active ds_transition to Deactivated
Sep 22 23:21:38.393 INFO [0] Transition from Active to Deactivated
Sep 22 23:21:38.393 INFO [1] check deactivate YES
Sep 22 23:21:38.393 INFO [1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (02fbce88-f31f-4b2e-a8ed-d7530615b97b) Deactivated Active Active ds_transition to Deactivated
Sep 22 23:21:38.393 INFO [1] Transition from Active to Deactivated
Sep 22 23:21:38.393 DEBG [2] deactivate flush 1001 done, : downstairs
Sep 22 23:21:38.393 INFO [2] check deactivate YES
Sep 22 23:21:38.393 INFO [2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (02fbce88-f31f-4b2e-a8ed-d7530615b97b) Deactivated Deactivated Active ds_transition to Deactivated
Sep 22 23:21:38.393 INFO [2] Transition from Active to Deactivated
Sep 22 23:21:38.393 ERRO 127.0.0.1:65363: proc: [0] client work task ended, Ok(Err([0] exits after deactivation)), so we end too, looper: 0
Sep 22 23:21:38.393 INFO [0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Gone missing, transition from Deactivated to New
Sep 22 23:21:38.393 INFO deactivate transition checking...
Sep 22 23:21:38.393 INFO deactivate_transition New Maybe 
Sep 22 23:21:38.393 INFO deactivate_transition Deactivated NO
Sep 22 23:21:38.393 INFO deactivate_transition Deactivated NO
Sep 22 23:21:38.393 INFO [0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 connection to 127.0.0.1:65363 closed, looper: 0
Sep 22 23:21:38.393 ERRO 127.0.0.1:63095: proc: [1] client work task ended, Ok(Err([1] exits after deactivation)), so we end too, looper: 1
Sep 22 23:21:38.393 INFO [1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Gone missing, transition from Deactivated to New
Sep 22 23:21:38.393 INFO deactivate transition checking...
Sep 22 23:21:38.393 INFO deactivate_transition New Maybe 
Sep 22 23:21:38.393 INFO deactivate_transition New Maybe 
Sep 22 23:21:38.393 INFO deactivate_transition Deactivated NO
Sep 22 23:21:38.393 INFO [1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 connection to 127.0.0.1:63095 closed, looper: 1
Sep 22 23:21:38.394 DEBG up_ds_listen was notified
Sep 22 23:21:38.394 DEBG up_ds_listen process 1001
Sep 22 23:21:38.394 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.394 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:21:38.394 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.394 ERRO 127.0.0.1:64907: proc: [2] client work task ended, Ok(Err([2] exits after deactivation)), so we end too, looper: 2
Sep 22 23:21:38.394 INFO [2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Gone missing, transition from Deactivated to New
Sep 22 23:21:38.394 INFO deactivate transition checking...
Sep 22 23:21:38.394 INFO deactivate_transition New Maybe 
Sep 22 23:21:38.394 INFO deactivate_transition New Maybe 
Sep 22 23:21:38.394 INFO deactivate_transition New Maybe 
Sep 22 23:21:38.394 INFO All DS in the proper state! -> INIT
Sep 22 23:21:38.394 INFO [2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 connection to 127.0.0.1:64907 closed, looper: 2
Sep 22 23:21:38.394 INFO [0] 127.0.0.1:65363 task reports connection:false
Sep 22 23:21:38.394 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 New New New
Sep 22 23:21:38.394 INFO [0] 127.0.0.1:65363 task reports offline
Sep 22 23:21:38.394 INFO [1] 127.0.0.1:63095 task reports connection:false
Sep 22 23:21:38.394 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 New New New
Sep 22 23:21:38.394 INFO [1] 127.0.0.1:63095 task reports offline
Sep 22 23:21:38.394 INFO [2] 127.0.0.1:64907 task reports connection:false
Sep 22 23:21:38.394 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 New New New
Sep 22 23:21:38.394 INFO [2] 127.0.0.1:64907 task reports offline
note: configured to log to "/dev/stdout"
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.394925867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb/bulk_write","method":"POST","req_id":"e522cfdb-4708-4a23-858e-1d0d5f323626","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":1110,"response_code":"204"}
Sep 22 23:21:38.395 DEBG Write     :1003 deps:[] res:true
Sep 22 23:21:38.395 DEBG Write     :1003 deps:[] res:true
Sep 22 23:21:38.396 DEBG Write     :1003 deps:[] res:true
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.396513554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63903","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:63903","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.396680499Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.398252645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb/bulk_write","method":"POST","req_id":"297166ba-ec88-4c99-9a90-f12675f9a285","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":1405,"response_code":"204"}
Sep 22 23:21:38.398 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.399 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.399 INFO Opened existing region file "/tmp/downstairs-XotxiwRu/region.json"
Sep 22 23:21:38.399 INFO Database read version 1
Sep 22 23:21:38.399 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:38.399 INFO Database write version 1
Sep 22 23:21:38.399 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:38.399 DEBG Write     :1004 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.401951711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb/bulk_write","method":"POST","req_id":"47e56d17-6c0f-4338-9a8d-1862a6e302ed","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":1440,"response_code":"204"}
Sep 22 23:21:38.402 DEBG Write     :1005 deps:[] res:true
Sep 22 23:21:38.403 DEBG Write     :1005 deps:[] res:true
Sep 22 23:21:38.403 DEBG Write     :1005 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.406174257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb/bulk_write","method":"POST","req_id":"d622fb87-a0dd-4ceb-80ff-9b2f1a9dfed6","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":1404,"response_code":"204"}
Sep 22 23:21:38.406 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:38.406 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:38.407 DEBG Write     :1006 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.408754753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb/bulk_write","method":"POST","req_id":"b1177d14-db18-4b39-9091-3cd7cce02134","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":1042,"response_code":"204"}
Sep 22 23:21:38.409 DEBG Write     :1007 deps:[] res:true
Sep 22 23:21:38.409 DEBG Write     :1007 deps:[] res:true
Sep 22 23:21:38.409 DEBG Write     :1007 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.411496777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb/bulk_write","method":"POST","req_id":"451fdf79-77d7-4af9-a4c4-3ade94f01481","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":986,"response_code":"204"}
Sep 22 23:21:38.411 DEBG Write     :1008 deps:[] res:true
Sep 22 23:21:38.412 DEBG Write     :1008 deps:[] res:true
Sep 22 23:21:38.412 DEBG Write     :1008 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.414322902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb/bulk_write","method":"POST","req_id":"94442a37-08c7-4cc8-9eff-16d447b5f2c9","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":1002,"response_code":"204"}
Sep 22 23:21:38.414 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:38.414 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:38.415 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:38.415 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.415 INFO Created new region file "/tmp/downstairs-hwOOuWZz/region.json"
{"msg":"detach removing entry for volume 17fca985-4bd4-4289-96a4-634e1d483fcb","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.415844363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume 17fca985-4bd4-4289-96a4-634e1d483fcb","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.415898951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.418 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.419 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.419 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.419600456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.419655448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.419897976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/17fca985-4bd4-4289-96a4-634e1d483fcb","method":"DELETE","req_id":"35cb97e1-bffa-4a4f-98e1-d29d3d140c7d","remote_addr":"127.0.0.1:43786","local_addr":"127.0.0.1:55883","component":"dropshot","latency_us":4147,"response_code":"204"}
Sep 22 23:21:38.420 INFO Upstairs starts
Sep 22 23:21:38.420 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.420 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.420 INFO Crucible stats registered with UUID: e5b4ada7-7239-4d69-8861-23d72de6320f
Sep 22 23:21:38.420 INFO Crucible e5b4ada7-7239-4d69-8861-23d72de6320f has session id: 592327dc-1ee1-4d46-9c50-69a4560944c7
Sep 22 23:21:38.420 INFO [0] connecting to 127.0.0.1:33527, looper: 0
Sep 22 23:21:38.420 INFO [1] connecting to 127.0.0.1:59188, looper: 1
Sep 22 23:21:38.420 INFO [2] connecting to 127.0.0.1:54900, looper: 2
Sep 22 23:21:38.420 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.420 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.421 INFO Flush timeout: 0.5
Sep 22 23:21:38.421 INFO accepted connection from 127.0.0.1:61046, task: main
Sep 22 23:21:38.421 INFO accepted connection from 127.0.0.1:52775, task: main
Sep 22 23:21:38.421 INFO accepted connection from 127.0.0.1:55794, task: main
Sep 22 23:21:38.421 INFO [0] e5b4ada7-7239-4d69-8861-23d72de6320f looper connected, looper: 0
Sep 22 23:21:38.421 INFO [0] Proc runs for 127.0.0.1:33527 in state New
Sep 22 23:21:38.421 INFO [1] e5b4ada7-7239-4d69-8861-23d72de6320f looper connected, looper: 1
Sep 22 23:21:38.421 INFO [1] Proc runs for 127.0.0.1:59188 in state New
Sep 22 23:21:38.421 INFO [2] e5b4ada7-7239-4d69-8861-23d72de6320f looper connected, looper: 2
Sep 22 23:21:38.421 INFO [2] Proc runs for 127.0.0.1:54900 in state New
Sep 22 23:21:38.422 INFO Connection request from e5b4ada7-7239-4d69-8861-23d72de6320f with version 4, task: proc
Sep 22 23:21:38.422 INFO upstairs UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:38.422 INFO Connection request from e5b4ada7-7239-4d69-8861-23d72de6320f with version 4, task: proc
Sep 22 23:21:38.422 INFO upstairs UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:38.422 INFO Connection request from e5b4ada7-7239-4d69-8861-23d72de6320f with version 4, task: proc
Sep 22 23:21:38.422 INFO upstairs UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:38.422 INFO [0] e5b4ada7-7239-4d69-8861-23d72de6320f (d15f82d8-86d1-4a0a-9592-c9a50e119254) New New New ds_transition to WaitActive
Sep 22 23:21:38.422 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.422 INFO [1] e5b4ada7-7239-4d69-8861-23d72de6320f (d15f82d8-86d1-4a0a-9592-c9a50e119254) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.422 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.422 INFO [2] e5b4ada7-7239-4d69-8861-23d72de6320f (d15f82d8-86d1-4a0a-9592-c9a50e119254) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.422 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:21:38.423 INFO e5b4ada7-7239-4d69-8861-23d72de6320f active request set
Sep 22 23:21:38.423 INFO [0] received activate with gen 2
Sep 22 23:21:38.423 INFO [0] client got ds_active_rx, promote! session d15f82d8-86d1-4a0a-9592-c9a50e119254
Sep 22 23:21:38.423 INFO [1] received activate with gen 2
Sep 22 23:21:38.423 INFO [1] client got ds_active_rx, promote! session d15f82d8-86d1-4a0a-9592-c9a50e119254
Sep 22 23:21:38.423 INFO [2] received activate with gen 2
Sep 22 23:21:38.423 INFO [2] client got ds_active_rx, promote! session d15f82d8-86d1-4a0a-9592-c9a50e119254
Sep 22 23:21:38.423 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } to UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 }
Sep 22 23:21:38.423 WARN Signaling to UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } thread that UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } is being promoted (read-write)
Sep 22 23:21:38.423 INFO UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } is now active (read-write)
Sep 22 23:21:38.423 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } to UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 }
Sep 22 23:21:38.423 WARN Signaling to UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } thread that UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } is being promoted (read-write)
Sep 22 23:21:38.423 INFO UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } is now active (read-write)
Sep 22 23:21:38.423 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } to UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 }
Sep 22 23:21:38.423 WARN Signaling to UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 } thread that UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } is being promoted (read-write)
Sep 22 23:21:38.424 INFO UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } is now active (read-write)
Sep 22 23:21:38.424 WARN Another upstairs UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 }, task: main
Sep 22 23:21:38.424 INFO connection (127.0.0.1:55430): all done
Sep 22 23:21:38.424 WARN Another upstairs UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 }, task: main
Sep 22 23:21:38.424 INFO connection (127.0.0.1:50291): all done
Sep 22 23:21:38.424 WARN Another upstairs UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: d15f82d8-86d1-4a0a-9592-c9a50e119254, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: e5b4ada7-7239-4d69-8861-23d72de6320f, session_id: 22467938-6385-426f-86c0-fa56c94d98b1, gen: 1 }, task: main
Sep 22 23:21:38.424 INFO connection (127.0.0.1:55513): all done
{"msg":"[1] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) cmd_loop saw YouAreNoLongerActive e5b4ada7-7239-4d69-8861-23d72de6320f d15f82d8-86d1-4a0a-9592-c9a50e119254 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.424818687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) Active Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.424878408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.424916188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f set inactive, session 22467938-6385-426f-86c0-fa56c94d98b1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.424950293Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:59188: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.424984295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] e5b4ada7-7239-4d69-8861-23d72de6320f Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425024807Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] e5b4ada7-7239-4d69-8861-23d72de6320f connection to 127.0.0.1:59188 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425069555Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) cmd_loop saw YouAreNoLongerActive e5b4ada7-7239-4d69-8861-23d72de6320f d15f82d8-86d1-4a0a-9592-c9a50e119254 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.425122908Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) Active Disconnected Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425167026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425207306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f set inactive, session 22467938-6385-426f-86c0-fa56c94d98b1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425251195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:54900: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.425294919Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] e5b4ada7-7239-4d69-8861-23d72de6320f Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425336464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] e5b4ada7-7239-4d69-8861-23d72de6320f connection to 127.0.0.1:54900 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425377724Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[0] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) cmd_loop saw YouAreNoLongerActive e5b4ada7-7239-4d69-8861-23d72de6320f d15f82d8-86d1-4a0a-9592-c9a50e119254 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.425429147Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] e5b4ada7-7239-4d69-8861-23d72de6320f (22467938-6385-426f-86c0-fa56c94d98b1) Active Disconnected Disconnected ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425476375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425518379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f set inactive, session 22467938-6385-426f-86c0-fa56c94d98b1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425560788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:33527: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.425602875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] e5b4ada7-7239-4d69-8861-23d72de6320f Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425649283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] e5b4ada7-7239-4d69-8861-23d72de6320f connection to 127.0.0.1:33527 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425700056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.425746869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.425 INFO [0] downstairs client at 127.0.0.1:33527 has UUID c25278bf-5b9e-4380-abf6-dfd9701dce27
{"msg":"[1] 127.0.0.1:59188 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425786937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.425 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c25278bf-5b9e-4380-abf6-dfd9701dce27, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425827689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.425 INFO e5b4ada7-7239-4d69-8861-23d72de6320f WaitActive WaitActive WaitActive
{"msg":"[1] 127.0.0.1:59188 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425866569Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:54900 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.425907609Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.425 INFO [1] downstairs client at 127.0.0.1:59188 has UUID 99808fa8-ed0d-4880-abf3-c77623f8cd97
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.4259837Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.425 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 99808fa8-ed0d-4880-abf3-c77623f8cd97, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[2] 127.0.0.1:54900 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.426025393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.426 INFO e5b4ada7-7239-4d69-8861-23d72de6320f WaitActive WaitActive WaitActive
{"msg":"[0] 127.0.0.1:33527 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.426063318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"e5b4ada7-7239-4d69-8861-23d72de6320f Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.426092523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:33527 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.426129955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.426 INFO [2] downstairs client at 127.0.0.1:54900 has UUID 651c2c7a-b2ed-48c3-b57d-44ab91a6f440
Sep 22 23:21:38.426 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 651c2c7a-b2ed-48c3-b57d-44ab91a6f440, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[2] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.426171026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.426 INFO e5b4ada7-7239-4d69-8861-23d72de6320f WaitActive WaitActive WaitActive
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.426215397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.426 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:38.426 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.427 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:38.427 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.427 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:38.427 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.428 INFO [0] e5b4ada7-7239-4d69-8861-23d72de6320f (d15f82d8-86d1-4a0a-9592-c9a50e119254) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.428 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.428 WARN [0] new RM replaced this: None
Sep 22 23:21:38.428 INFO [0] Starts reconcile loop
Sep 22 23:21:38.428 INFO [1] e5b4ada7-7239-4d69-8861-23d72de6320f (d15f82d8-86d1-4a0a-9592-c9a50e119254) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.428 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.428 WARN [1] new RM replaced this: None
Sep 22 23:21:38.428 INFO [1] Starts reconcile loop
Sep 22 23:21:38.428 INFO [2] e5b4ada7-7239-4d69-8861-23d72de6320f (d15f82d8-86d1-4a0a-9592-c9a50e119254) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.428 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.428 WARN [2] new RM replaced this: None
Sep 22 23:21:38.428 INFO [2] Starts reconcile loop
Sep 22 23:21:38.428 INFO [0] 127.0.0.1:33527 task reports connection:true
Sep 22 23:21:38.428 INFO e5b4ada7-7239-4d69-8861-23d72de6320f WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.428 INFO [0]R  flush_numbers: [1, 1]
Sep 22 23:21:38.428 INFO [0]R  generation: [1, 1]
Sep 22 23:21:38.428 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.428 INFO [1]R  flush_numbers: [1, 1]
Sep 22 23:21:38.428 INFO [1]R  generation: [1, 1]
Sep 22 23:21:38.428 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.428 INFO [2]R  flush_numbers: [1, 1]
Sep 22 23:21:38.428 INFO [2]R  generation: [1, 1]
Sep 22 23:21:38.428 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.428 INFO Max found gen is 2
Sep 22 23:21:38.428 INFO Generation requested: 2 >= found:2
Sep 22 23:21:38.428 INFO Next flush: 2
Sep 22 23:21:38.428 INFO All extents match
Sep 22 23:21:38.428 INFO No downstairs repair required
Sep 22 23:21:38.428 INFO No initial repair work was required
Sep 22 23:21:38.428 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.428 INFO e5b4ada7-7239-4d69-8861-23d72de6320f is now active with session: d15f82d8-86d1-4a0a-9592-c9a50e119254
Sep 22 23:21:38.428 INFO e5b4ada7-7239-4d69-8861-23d72de6320f Set Active after no repair
Sep 22 23:21:38.428 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.429 INFO Set check for repair
Sep 22 23:21:38.429 INFO [1] 127.0.0.1:59188 task reports connection:true
Sep 22 23:21:38.429 INFO e5b4ada7-7239-4d69-8861-23d72de6320f Active Active Active
Sep 22 23:21:38.429 INFO Set check for repair
Sep 22 23:21:38.429 INFO [2] 127.0.0.1:54900 task reports connection:true
Sep 22 23:21:38.429 INFO e5b4ada7-7239-4d69-8861-23d72de6320f Active Active Active
Sep 22 23:21:38.429 INFO Set check for repair
Sep 22 23:21:38.429 INFO [0] received reconcile message
Sep 22 23:21:38.429 INFO [0] All repairs completed, exit
Sep 22 23:21:38.429 INFO [0] Starts cmd_loop
Sep 22 23:21:38.429 INFO [1] received reconcile message
Sep 22 23:21:38.429 INFO [1] All repairs completed, exit
Sep 22 23:21:38.429 INFO [1] Starts cmd_loop
Sep 22 23:21:38.429 INFO [2] received reconcile message
Sep 22 23:21:38.429 INFO [2] All repairs completed, exit
Sep 22 23:21:38.429 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.430 DEBG IO Read  1000 has deps []
Sep 22 23:21:38.431 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.431 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.432 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.436 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:38.438 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:38.441 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:38.441 DEBG up_ds_listen was notified
Sep 22 23:21:38.441 DEBG up_ds_listen process 1000
Sep 22 23:21:38.441 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.441 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::test_pantry_bulk_write ... ok
Sep 22 23:21:38.448 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.448 INFO Created new region file "/tmp/downstairs-rDPgnXdu/region.json"
Sep 22 23:21:38.461 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:21:38.464 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.464 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.464 DEBG up_ds_listen was notified
Sep 22 23:21:38.464 DEBG up_ds_listen process 1001
Sep 22 23:21:38.464 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:38.464 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:21:38.464 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:38.483 WARN upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:21:38.483 WARN upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } was previously active, clearing, task: main
Sep 22 23:21:38.484 INFO connection (127.0.0.1:46576): all done
Sep 22 23:21:38.484 WARN upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:21:38.484 WARN upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } was previously active, clearing, task: main
Sep 22 23:21:38.484 INFO connection (127.0.0.1:61396): all done
Sep 22 23:21:38.484 WARN upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:21:38.484 WARN upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 02fbce88-f31f-4b2e-a8ed-d7530615b97b, gen: 1 } was previously active, clearing, task: main
Sep 22 23:21:38.484 INFO connection (127.0.0.1:56109): all done
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.484876697Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63903","component":"dropshot","remote_addr":"127.0.0.1:56972"}
{"msg":"no entry exists for volume 6a15a37b-9c1f-4533-aac2-9a0f3a29490f, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.485691133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.485952702Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.486001536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.486032387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: f5cf38e1-07a4-4403-b783-c67b3e5d3b37","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.486062415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible f5cf38e1-07a4-4403-b783-c67b3e5d3b37 has session id: 30a44f31-a40d-49e5-9fd4-5bd01d26e4ac","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.486091364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:65363","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.486248871Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:63095","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.486387631Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:64907","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.4865206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.486631181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.486662325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.48668467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 6a15a37b-9c1f-4533-aac2-9a0f3a29490f constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.486879809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.487119498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.487 INFO accepted connection from 127.0.0.1:50295, task: main
Sep 22 23:21:38.487 INFO accepted connection from 127.0.0.1:60639, task: main
Sep 22 23:21:38.487 INFO accepted connection from 127.0.0.1:55335, task: main
{"msg":"[0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.487697405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:65363 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.487745982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.487789291Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:63095 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.487837664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.487883635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
Sep 22 23:21:38.487 INFO Connection request from f5cf38e1-07a4-4403-b783-c67b3e5d3b37 with version 4, task: proc
{"msg":"[2] Proc runs for 127.0.0.1:64907 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.487936947Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.487 INFO upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.488 INFO Connection request from f5cf38e1-07a4-4403-b783-c67b3e5d3b37 with version 4, task: proc
Sep 22 23:21:38.488 INFO upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.488 INFO Connection request from f5cf38e1-07a4-4403-b783-c67b3e5d3b37 with version 4, task: proc
Sep 22 23:21:38.488 INFO upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } connected, version 4, task: proc
{"msg":"[0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.488384838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.488424682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 9ca4c47d-0d69-4204-a6cd-e8c5971753b4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.488467215Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.488508765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.488548163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 9ca4c47d-0d69-4204-a6cd-e8c5971753b4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.488589499Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.48863145Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.488671673Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.488 INFO UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } is now active (read-write)
{"msg":"[2] client is_active_req TRUE, promote! session 9ca4c47d-0d69-4204-a6cd-e8c5971753b4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.4887122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.488 INFO UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } is now active (read-write)
Sep 22 23:21:38.488 INFO UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:65363 has UUID ad667607-b05e-4427-9709-433732d1d5ab","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.489437166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ad667607-b05e-4427-9709-433732d1d5ab, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.48948144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.4895066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:63095 has UUID 995b5b97-67dd-43f0-b20a-d5b2c0c00924","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.489530557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 995b5b97-67dd-43f0-b20a-d5b2c0c00924, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.489557999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.489581384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:64907 has UUID f0f4ef77-64c9-4cae-ad93-ac2fb4943d50","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.489605741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f0f4ef77-64c9-4cae-ad93-ac2fb4943d50, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.489632929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.489656276Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.489 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.490 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.490 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.490 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.491 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.491 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.491978698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492009682Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.49203262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492060195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492086375Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492109182Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.492131333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492153574Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492178156Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.49220052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.49222271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492244319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:65363 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.49226727Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492290392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492320624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492344079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492365788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492387662Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492409244Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492430984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492453511Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492479595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492502336Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492524388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492546338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492567831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492589366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492611154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492633042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492654974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 is now active with session: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492678364Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492701141Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492723534Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492745191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:63095 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492767477Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492789981Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492811578Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:64907 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492833892Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492856494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492878358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492904159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492927086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492956884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.492983694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.493010271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.493032338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.493054144Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.493076098Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.493097495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 6a15a37b-9c1f-4533-aac2-9a0f3a29490f activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.493119846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 6a15a37b-9c1f-4533-aac2-9a0f3a29490f constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.493142857Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.493164374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/6a15a37b-9c1f-4533-aac2-9a0f3a29490f","method":"POST","req_id":"a451b240-4668-432f-bef0-04505eeedc8a","remote_addr":"127.0.0.1:56972","local_addr":"127.0.0.1:63903","component":"dropshot","latency_us":7602,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.494173219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/6a15a37b-9c1f-4533-aac2-9a0f3a29490f/import_from_url","method":"POST","req_id":"3abc8a05-6de8-4254-99e3-103189fc2dd1","remote_addr":"127.0.0.1:56972","local_addr":"127.0.0.1:63903","component":"dropshot","latency_us":414,"response_code":"200"}
Sep 22 23:21:38.507 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.507 INFO Opened existing region file "/tmp/downstairs-sxN8iMqh/region.json"
Sep 22 23:21:38.507 INFO Database read version 1
Sep 22 23:21:38.507 INFO Database write version 1
Sep 22 23:21:38.553 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.553 INFO Opened existing region file "/tmp/downstairs-9KQd6X6V/region.json"
Sep 22 23:21:38.553 INFO Database read version 1
Sep 22 23:21:38.553 INFO Database write version 1
Sep 22 23:21:38.584 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.585 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.586 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:38.588 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.588 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:38.588 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.59105337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/1df59235-070a-40ea-abd7-4364f8871e04/ok","method":"GET","req_id":"7bb2c307-3a70-4afb-9141-62fda82d9cfa","remote_addr":"127.0.0.1:56972","local_addr":"127.0.0.1:63903","component":"dropshot","latency_us":12975,"response_code":"200"}
{"msg":"detach removing entry for volume 6a15a37b-9c1f-4533-aac2-9a0f3a29490f","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.59180079Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume 6a15a37b-9c1f-4533-aac2-9a0f3a29490f","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.591856353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.592 DEBG Flush     :1002 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:21:38.592 DEBG Flush     :1002 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:21:38.592 DEBG Flush     :1002 extent_limit None deps:[] res:true f:2 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.593237166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.593290627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.593546503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/6a15a37b-9c1f-4533-aac2-9a0f3a29490f","method":"DELETE","req_id":"b7960ba4-c817-4103-b2b5-de050300c968","remote_addr":"127.0.0.1:56972","local_addr":"127.0.0.1:63903","component":"dropshot","latency_us":1852,"response_code":"204"}
Sep 22 23:21:38.593 INFO Upstairs starts
Sep 22 23:21:38.594 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:38.594 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:38.594 INFO Crucible stats registered with UUID: f5cf38e1-07a4-4403-b783-c67b3e5d3b37
Sep 22 23:21:38.594 INFO Crucible f5cf38e1-07a4-4403-b783-c67b3e5d3b37 has session id: 22525190-99f3-4342-b603-249740a74d31
Sep 22 23:21:38.594 INFO [0] connecting to 127.0.0.1:65363, looper: 0
Sep 22 23:21:38.594 INFO [1] connecting to 127.0.0.1:63095, looper: 1
Sep 22 23:21:38.594 INFO [2] connecting to 127.0.0.1:64907, looper: 2
Sep 22 23:21:38.594 INFO up_listen starts, task: up_listen
Sep 22 23:21:38.594 INFO Wait for all three downstairs to come online
Sep 22 23:21:38.594 INFO Flush timeout: 0.5
Sep 22 23:21:38.594 INFO accepted connection from 127.0.0.1:51063, task: main
Sep 22 23:21:38.594 INFO accepted connection from 127.0.0.1:35409, task: main
Sep 22 23:21:38.594 INFO accepted connection from 127.0.0.1:47713, task: main
Sep 22 23:21:38.595 INFO [0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 looper connected, looper: 0
Sep 22 23:21:38.595 INFO [0] Proc runs for 127.0.0.1:65363 in state New
Sep 22 23:21:38.595 INFO [1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 looper connected, looper: 1
Sep 22 23:21:38.595 INFO [1] Proc runs for 127.0.0.1:63095 in state New
Sep 22 23:21:38.595 INFO [2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 looper connected, looper: 2
Sep 22 23:21:38.595 INFO [2] Proc runs for 127.0.0.1:64907 in state New
Sep 22 23:21:38.595 INFO Connection request from f5cf38e1-07a4-4403-b783-c67b3e5d3b37 with version 4, task: proc
Sep 22 23:21:38.595 INFO upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:38.595 INFO Connection request from f5cf38e1-07a4-4403-b783-c67b3e5d3b37 with version 4, task: proc
Sep 22 23:21:38.595 INFO upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:38.595 INFO Connection request from f5cf38e1-07a4-4403-b783-c67b3e5d3b37 with version 4, task: proc
Sep 22 23:21:38.595 INFO upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:38.595 INFO [0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (e5eef348-dcf9-4634-aa8e-08126b692f43) New New New ds_transition to WaitActive
Sep 22 23:21:38.595 INFO [0] Transition from New to WaitActive
Sep 22 23:21:38.595 INFO [1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (e5eef348-dcf9-4634-aa8e-08126b692f43) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:38.595 INFO [1] Transition from New to WaitActive
Sep 22 23:21:38.596 INFO [2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (e5eef348-dcf9-4634-aa8e-08126b692f43) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:38.596 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:21:38.596 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 active request set
Sep 22 23:21:38.596 INFO [0] received activate with gen 3
Sep 22 23:21:38.596 INFO [0] client got ds_active_rx, promote! session e5eef348-dcf9-4634-aa8e-08126b692f43
Sep 22 23:21:38.596 INFO [1] received activate with gen 3
Sep 22 23:21:38.596 INFO [1] client got ds_active_rx, promote! session e5eef348-dcf9-4634-aa8e-08126b692f43
Sep 22 23:21:38.596 INFO [2] received activate with gen 3
Sep 22 23:21:38.596 INFO [2] client got ds_active_rx, promote! session e5eef348-dcf9-4634-aa8e-08126b692f43
Sep 22 23:21:38.596 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } to UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 }
Sep 22 23:21:38.596 WARN Signaling to UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } thread that UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } is being promoted (read-write)
Sep 22 23:21:38.596 INFO UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } is now active (read-write)
Sep 22 23:21:38.596 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } to UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 }
Sep 22 23:21:38.596 WARN Signaling to UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } thread that UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } is being promoted (read-write)
Sep 22 23:21:38.596 INFO UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } is now active (read-write)
Sep 22 23:21:38.596 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } to UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 }
Sep 22 23:21:38.596 WARN Signaling to UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 } thread that UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } is being promoted (read-write)
Sep 22 23:21:38.596 INFO UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } is now active (read-write)
Sep 22 23:21:38.596 WARN Another upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 }, task: main
Sep 22 23:21:38.596 INFO connection (127.0.0.1:50295): all done
Sep 22 23:21:38.596 WARN Another upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 }, task: main
Sep 22 23:21:38.597 INFO connection (127.0.0.1:60639): all done
Sep 22 23:21:38.597 WARN Another upstairs UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: e5eef348-dcf9-4634-aa8e-08126b692f43, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: f5cf38e1-07a4-4403-b783-c67b3e5d3b37, session_id: 9ca4c47d-0d69-4204-a6cd-e8c5971753b4, gen: 1 }, task: main
Sep 22 23:21:38.597 INFO connection (127.0.0.1:55335): all done
{"msg":"[0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) cmd_loop saw YouAreNoLongerActive f5cf38e1-07a4-4403-b783-c67b3e5d3b37 e5eef348-dcf9-4634-aa8e-08126b692f43 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.597450449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) Active Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.59750324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.597545245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 set inactive, session 9ca4c47d-0d69-4204-a6cd-e8c5971753b4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.597590208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:65363: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.597632351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.597677122Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 connection to 127.0.0.1:65363 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.597711254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) cmd_loop saw YouAreNoLongerActive f5cf38e1-07a4-4403-b783-c67b3e5d3b37 e5eef348-dcf9-4634-aa8e-08126b692f43 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.597762933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) Disconnected Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.597807677Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.597845885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 set inactive, session 9ca4c47d-0d69-4204-a6cd-e8c5971753b4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.597887229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:63095: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.597932728Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.597979378Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 connection to 127.0.0.1:63095 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598021191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) cmd_loop saw YouAreNoLongerActive f5cf38e1-07a4-4403-b783-c67b3e5d3b37 e5eef348-dcf9-4634-aa8e-08126b692f43 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.598069776Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (9ca4c47d-0d69-4204-a6cd-e8c5971753b4) Disconnected Disconnected Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598121223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598163965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 set inactive, session 9ca4c47d-0d69-4204-a6cd-e8c5971753b4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598208554Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.598 INFO [0] downstairs client at 127.0.0.1:65363 has UUID ad667607-b05e-4427-9709-433732d1d5ab
{"msg":"127.0.0.1:64907: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:38.598252297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
Sep 22 23:21:38.598 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: ad667607-b05e-4427-9709-433732d1d5ab, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598297419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.598 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitActive WaitActive WaitActive
{"msg":"[2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 connection to 127.0.0.1:64907 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598344999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.598385584Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.598 INFO [1] downstairs client at 127.0.0.1:63095 has UUID 995b5b97-67dd-43f0-b20a-d5b2c0c00924
Sep 22 23:21:38.598 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 995b5b97-67dd-43f0-b20a-d5b2c0c00924, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[0] 127.0.0.1:65363 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598413456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598437354Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.598 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitActive WaitActive WaitActive
{"msg":"[0] 127.0.0.1:65363 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598474236Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.598 INFO [2] downstairs client at 127.0.0.1:64907 has UUID f0f4ef77-64c9-4cae-ad93-ac2fb4943d50
{"msg":"[1] 127.0.0.1:63095 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598509408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.598 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f0f4ef77-64c9-4cae-ad93-ac2fb4943d50, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598542189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.598 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitActive WaitActive WaitActive
{"msg":"[1] 127.0.0.1:63095 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598571718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:64907 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598609163Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.598645488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:64907 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:38.59867611Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.598699491Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:38.598721321Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:38.598 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:38.598 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.599 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:38.599 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.599 INFO Current flush_numbers [0..12]: [1, 1]
Sep 22 23:21:38.599 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.599 INFO [0] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (e5eef348-dcf9-4634-aa8e-08126b692f43) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.599 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.599 WARN [0] new RM replaced this: None
Sep 22 23:21:38.599 INFO [0] Starts reconcile loop
Sep 22 23:21:38.599 INFO [1] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (e5eef348-dcf9-4634-aa8e-08126b692f43) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.599 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.599 WARN [1] new RM replaced this: None
Sep 22 23:21:38.599 INFO [1] Starts reconcile loop
Sep 22 23:21:38.599 INFO [2] f5cf38e1-07a4-4403-b783-c67b3e5d3b37 (e5eef348-dcf9-4634-aa8e-08126b692f43) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:38.599 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:38.599 WARN [2] new RM replaced this: None
Sep 22 23:21:38.599 INFO [2] Starts reconcile loop
Sep 22 23:21:38.599 INFO [0] 127.0.0.1:65363 task reports connection:true
Sep 22 23:21:38.599 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:38.599 INFO [0]R  flush_numbers: [1, 1]
Sep 22 23:21:38.599 INFO [0]R  generation: [1, 1]
Sep 22 23:21:38.599 INFO [0]R  dirty: [false, false]
Sep 22 23:21:38.599 INFO [1]R  flush_numbers: [1, 1]
Sep 22 23:21:38.599 INFO [1]R  generation: [1, 1]
Sep 22 23:21:38.599 INFO [1]R  dirty: [false, false]
Sep 22 23:21:38.599 INFO [2]R  flush_numbers: [1, 1]
Sep 22 23:21:38.599 INFO [2]R  generation: [1, 1]
Sep 22 23:21:38.599 INFO [2]R  dirty: [false, false]
Sep 22 23:21:38.599 INFO Max found gen is 2
Sep 22 23:21:38.599 INFO Generation requested: 3 >= found:2
Sep 22 23:21:38.599 INFO Next flush: 2
Sep 22 23:21:38.600 INFO All extents match
Sep 22 23:21:38.600 INFO No downstairs repair required
Sep 22 23:21:38.600 INFO No initial repair work was required
Sep 22 23:21:38.600 INFO Set Downstairs and Upstairs active
Sep 22 23:21:38.600 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 is now active with session: e5eef348-dcf9-4634-aa8e-08126b692f43
Sep 22 23:21:38.600 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Set Active after no repair
Sep 22 23:21:38.600 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:38.600 INFO Set check for repair
Sep 22 23:21:38.600 INFO [1] 127.0.0.1:63095 task reports connection:true
Sep 22 23:21:38.600 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Active Active Active
Sep 22 23:21:38.600 INFO Set check for repair
Sep 22 23:21:38.600 INFO [2] 127.0.0.1:64907 task reports connection:true
Sep 22 23:21:38.600 INFO f5cf38e1-07a4-4403-b783-c67b3e5d3b37 Active Active Active
Sep 22 23:21:38.600 INFO Set check for repair
Sep 22 23:21:38.600 INFO [0] received reconcile message
Sep 22 23:21:38.600 INFO [0] All repairs completed, exit
Sep 22 23:21:38.600 INFO [0] Starts cmd_loop
Sep 22 23:21:38.600 INFO [1] received reconcile message
Sep 22 23:21:38.600 INFO [1] All repairs completed, exit
Sep 22 23:21:38.600 INFO [1] Starts cmd_loop
Sep 22 23:21:38.600 INFO [2] received reconcile message
Sep 22 23:21:38.600 INFO [2] All repairs completed, exit
Sep 22 23:21:38.600 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:38.600 DEBG IO Read  1000 has deps []
Sep 22 23:21:38.601 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.601 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.602 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:38.605 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:38.607 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:38.609 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:38.609 DEBG up_ds_listen was notified
Sep 22 23:21:38.609 DEBG up_ds_listen process 1000
Sep 22 23:21:38.609 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:38.610 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::test_pantry_import_from_local_server ... ok
Sep 22 23:21:38.622 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.622 INFO Created new region file "/tmp/downstairs-wuCOh34x/region.json"
Sep 22 23:21:38.707 INFO Checking if live repair is needed
Sep 22 23:21:38.707 INFO No Live Repair required at this time
Sep 22 23:21:38.725 INFO accepted connection from 127.0.0.1:36068, task: main
Sep 22 23:21:38.725 INFO [0] f712941d-0577-43e4-8a2e-7814ec270c09 looper connected, looper: 0
Sep 22 23:21:38.725 INFO [0] Proc runs for 127.0.0.1:47135 in state Replaced
Sep 22 23:21:38.725 INFO Connection request from f712941d-0577-43e4-8a2e-7814ec270c09 with version 4, task: proc
Sep 22 23:21:38.725 INFO upstairs UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.725 INFO [0] upstairs guest_io_ready=TRUE, promote! session 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae
Sep 22 23:21:38.726 INFO UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } is now active (read-write)
Sep 22 23:21:38.726 INFO [0] downstairs client at 127.0.0.1:47135 has UUID de5d1b92-54ba-463b-93eb-b2678e77643b
Sep 22 23:21:38.726 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: de5d1b92-54ba-463b-93eb-b2678e77643b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.726 WARN [0] replace downstairs uuid:ca34832d-be06-44a7-9364-f6c3bc44d7fb with de5d1b92-54ba-463b-93eb-b2678e77643b
Sep 22 23:21:38.726 INFO f712941d-0577-43e4-8a2e-7814ec270c09 Replaced Active Active
Sep 22 23:21:38.726 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.727 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.727 INFO [0] f712941d-0577-43e4-8a2e-7814ec270c09 (512f09eb-7e91-421e-9fa9-f1bb0acbe6ae) Replaced Active Active ds_transition to LiveRepairReady
Sep 22 23:21:38.727 INFO [0] Transition from Replaced to LiveRepairReady
Sep 22 23:21:38.727 WARN [0] new RM replaced this: None
Sep 22 23:21:38.727 WARN [0] f712941d-0577-43e4-8a2e-7814ec270c09 Enter Ready for LiveRepair mode
Sep 22 23:21:38.727 INFO [0] Starts cmd_loop
Sep 22 23:21:38.727 INFO [0] 127.0.0.1:47135 task reports connection:true
Sep 22 23:21:38.727 INFO f712941d-0577-43e4-8a2e-7814ec270c09 LiveRepairReady Active Active
Sep 22 23:21:38.727 INFO Set check for repair
Sep 22 23:21:38.745 INFO UUID: e8aacc37-8406-4fc6-b948-447b29fd08c0
Sep 22 23:21:38.745 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:38.745 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.745 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.745 INFO Using address: 127.0.0.1:36532, task: main
Sep 22 23:21:38.746 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.746 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:59354, task: repair
Sep 22 23:21:38.746 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:59354, task: repair
Sep 22 23:21:38.746 INFO listening, local_addr: 127.0.0.1:59354, task: repair
Sep 22 23:21:38.746 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:59354, task: repair
Sep 22 23:21:38.746 INFO Using repair address: 127.0.0.1:59354, task: main
Sep 22 23:21:38.746 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.771 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:38.771 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.772 INFO Opened existing region file "/tmp/downstairs-hwOOuWZz/region.json"
Sep 22 23:21:38.772 INFO Database read version 1
Sep 22 23:21:38.772 INFO Database write version 1
Sep 22 23:21:38.821 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.821 INFO Opened existing region file "/tmp/downstairs-rDPgnXdu/region.json"
Sep 22 23:21:38.821 INFO Database read version 1
Sep 22 23:21:38.821 INFO Database write version 1
Sep 22 23:21:38.828 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.829 INFO Created new region file "/tmp/downstairs-t7mPFuJ0/region.json"
Sep 22 23:21:38.896 INFO UUID: 6e2bf597-efa8-46d7-84b3-855cd0183cc5
Sep 22 23:21:38.896 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:38.896 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.896 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.896 INFO Using address: 127.0.0.1:33519, task: main
Sep 22 23:21:38.896 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.897 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44104, task: repair
Sep 22 23:21:38.897 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44104, task: repair
Sep 22 23:21:38.897 INFO listening, local_addr: 127.0.0.1:44104, task: repair
Sep 22 23:21:38.897 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44104, task: repair
Sep 22 23:21:38.897 INFO Using repair address: 127.0.0.1:44104, task: main
Sep 22 23:21:38.897 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.938 INFO UUID: aa05d4f9-364f-4612-a579-613b38785c79
Sep 22 23:21:38.938 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:38.938 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:38.938 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:38.938 INFO Using address: 127.0.0.1:37108, task: main
Sep 22 23:21:38.938 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:38.939 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53054, task: repair
Sep 22 23:21:38.939 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53054, task: repair
Sep 22 23:21:38.939 INFO listening, local_addr: 127.0.0.1:53054, task: repair
Sep 22 23:21:38.939 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53054, task: repair
Sep 22 23:21:38.939 INFO Using repair address: 127.0.0.1:53054, task: main
Sep 22 23:21:38.939 INFO No SSL acceptor configured, task: main
Sep 22 23:21:38.953 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.953 INFO Opened existing region file "/tmp/downstairs-wuCOh34x/region.json"
Sep 22 23:21:38.953 INFO Database read version 1
Sep 22 23:21:38.953 INFO Database write version 1
Sep 22 23:21:38.968 INFO Checking if live repair is needed
Sep 22 23:21:38.968 INFO No Live Repair required at this time
Sep 22 23:21:38.969 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.969 INFO Created new region file "/tmp/downstairs-VXdeuER3/region.json"
Sep 22 23:21:38.991 INFO accepted connection from 127.0.0.1:51097, task: main
Sep 22 23:21:38.991 INFO [0] 52fe8d40-333d-4b64-8663-2adf0476947f looper connected, looper: 0
Sep 22 23:21:38.992 INFO [0] Proc runs for 127.0.0.1:62901 in state Replaced
Sep 22 23:21:38.992 INFO Connection request from 52fe8d40-333d-4b64-8663-2adf0476947f with version 4, task: proc
Sep 22 23:21:38.992 INFO upstairs UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:38.992 INFO [0] upstairs guest_io_ready=TRUE, promote! session fc0a5f14-ae91-4f48-aded-c2a3fecef8a0
Sep 22 23:21:38.992 INFO UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } is now active (read-write)
Sep 22 23:21:38.992 INFO [0] downstairs client at 127.0.0.1:62901 has UUID f9881bf1-86b5-438d-b94a-613d437493c1
Sep 22 23:21:38.992 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: f9881bf1-86b5-438d-b94a-613d437493c1, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:38.992 WARN [0] replace downstairs uuid:da052c76-0745-44df-96e3-d0a8c019994b with f9881bf1-86b5-438d-b94a-613d437493c1
Sep 22 23:21:38.992 INFO 52fe8d40-333d-4b64-8663-2adf0476947f Replaced Active Active
Sep 22 23:21:38.993 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:38.993 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:38.993 INFO [0] 52fe8d40-333d-4b64-8663-2adf0476947f (fc0a5f14-ae91-4f48-aded-c2a3fecef8a0) Replaced Active Active ds_transition to LiveRepairReady
Sep 22 23:21:38.993 INFO [0] Transition from Replaced to LiveRepairReady
Sep 22 23:21:38.993 WARN [0] new RM replaced this: None
Sep 22 23:21:38.993 WARN [0] 52fe8d40-333d-4b64-8663-2adf0476947f Enter Ready for LiveRepair mode
Sep 22 23:21:38.993 INFO [0] Starts cmd_loop
Sep 22 23:21:38.994 INFO [0] 127.0.0.1:62901 task reports connection:true
Sep 22 23:21:38.994 INFO 52fe8d40-333d-4b64-8663-2adf0476947f LiveRepairReady Active Active
Sep 22 23:21:38.994 INFO Set check for repair
Sep 22 23:21:38.995 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:38.995 INFO Created new region file "/tmp/downstairs-UGCoHqWo/region.json"
Sep 22 23:21:39.085 INFO UUID: 0b1643bf-b244-4616-9246-4922ec6009af
Sep 22 23:21:39.085 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:39.085 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:39.085 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:39.085 INFO Using address: 127.0.0.1:48339, task: main
Sep 22 23:21:39.085 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:39.085 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:21:39.086 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:21:39.086 INFO listening, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:21:39.086 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:21:39.086 INFO Using repair address: 127.0.0.1:52864, task: main
Sep 22 23:21:39.086 INFO No SSL acceptor configured, task: main
Sep 22 23:21:39.120 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:39.121 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.121 INFO Opened existing region file "/tmp/downstairs-t7mPFuJ0/region.json"
Sep 22 23:21:39.121 INFO Database read version 1
Sep 22 23:21:39.121 INFO Database write version 1
Sep 22 23:21:39.149 INFO UUID: b740b481-b810-4686-a6ad-5e7a129e9669
Sep 22 23:21:39.150 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:39.150 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:39.150 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:39.150 INFO Using address: 127.0.0.1:55664, task: main
Sep 22 23:21:39.150 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:39.150 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40259, task: repair
Sep 22 23:21:39.150 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40259, task: repair
Sep 22 23:21:39.150 INFO listening, local_addr: 127.0.0.1:40259, task: repair
Sep 22 23:21:39.151 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40259, task: repair
Sep 22 23:21:39.151 INFO Using repair address: 127.0.0.1:40259, task: main
Sep 22 23:21:39.151 INFO No SSL acceptor configured, task: main
Sep 22 23:21:39.161 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.161 INFO Created new region file "/tmp/downstairs-zy0O1MvL/region.json"
Sep 22 23:21:39.234 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.234 INFO Created new region file "/tmp/downstairs-pHcECflQ/region.json"
Sep 22 23:21:39.243 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:39.244 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.244 INFO Opened existing region file "/tmp/downstairs-VXdeuER3/region.json"
Sep 22 23:21:39.244 INFO Database read version 1
Sep 22 23:21:39.244 INFO Database write version 1
Sep 22 23:21:39.247 INFO UUID: 1f0dc9eb-57e9-42b1-a5c8-0ed1810eb444
Sep 22 23:21:39.247 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:39.247 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:39.247 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:39.247 INFO Using address: 127.0.0.1:40057, task: main
Sep 22 23:21:39.248 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:39.248 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:59363, task: repair
Sep 22 23:21:39.248 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:59363, task: repair
Sep 22 23:21:39.248 INFO listening, local_addr: 127.0.0.1:59363, task: repair
Sep 22 23:21:39.248 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:59363, task: repair
Sep 22 23:21:39.248 INFO Using repair address: 127.0.0.1:59363, task: main
Sep 22 23:21:39.248 INFO No SSL acceptor configured, task: main
Sep 22 23:21:39.281 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:39.282 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.282 INFO Opened existing region file "/tmp/downstairs-UGCoHqWo/region.json"
Sep 22 23:21:39.282 INFO Database read version 1
Sep 22 23:21:39.282 INFO Database write version 1
Sep 22 23:21:39.326 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.326 INFO Created new region file "/tmp/downstairs-sK3WzHBh/region.json"
Sep 22 23:21:39.478 INFO UUID: 213a7130-8159-4409-8d01-9d7ac9bbe737
Sep 22 23:21:39.478 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:39.478 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:39.478 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:39.478 INFO Using address: 127.0.0.1:33670, task: main
Sep 22 23:21:39.479 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:39.479 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49396, task: repair
Sep 22 23:21:39.479 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49396, task: repair
Sep 22 23:21:39.479 INFO listening, local_addr: 127.0.0.1:49396, task: repair
Sep 22 23:21:39.479 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49396, task: repair
Sep 22 23:21:39.479 INFO Using repair address: 127.0.0.1:49396, task: main
Sep 22 23:21:39.479 INFO No SSL acceptor configured, task: main
Sep 22 23:21:39.497 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:39.498 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.498 INFO Opened existing region file "/tmp/downstairs-zy0O1MvL/region.json"
Sep 22 23:21:39.498 INFO Database read version 1
Sep 22 23:21:39.498 INFO Database write version 1
Sep 22 23:21:39.567 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:39.567 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.567 INFO Opened existing region file "/tmp/downstairs-pHcECflQ/region.json"
Sep 22 23:21:39.567 INFO Database read version 1
Sep 22 23:21:39.567 INFO Database write version 1
Sep 22 23:21:39.571 INFO Upstairs starts
Sep 22 23:21:39.571 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:39.571 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:39.571 INFO Crucible stats registered with UUID: b937f86b-985c-4e08-8b98-07f7aae5f860
Sep 22 23:21:39.571 INFO Crucible b937f86b-985c-4e08-8b98-07f7aae5f860 has session id: 62cf098e-ed8c-438a-a15a-59c67d1d98af
Sep 22 23:21:39.572 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:39.572 INFO [0] connecting to 127.0.0.1:36172, looper: 0
Sep 22 23:21:39.572 INFO [1] connecting to 127.0.0.1:36532, looper: 1
Sep 22 23:21:39.572 INFO [2] connecting to 127.0.0.1:33670, looper: 2
Sep 22 23:21:39.572 INFO up_listen starts, task: up_listen
Sep 22 23:21:39.572 INFO Wait for all three downstairs to come online
Sep 22 23:21:39.572 INFO Flush timeout: 0.5
Sep 22 23:21:39.572 INFO accepted connection from 127.0.0.1:63095, task: main
Sep 22 23:21:39.572 INFO accepted connection from 127.0.0.1:33942, task: main
Sep 22 23:21:39.572 INFO accepted connection from 127.0.0.1:47489, task: main
Sep 22 23:21:39.572 INFO [0] b937f86b-985c-4e08-8b98-07f7aae5f860 looper connected, looper: 0
Sep 22 23:21:39.573 INFO [0] Proc runs for 127.0.0.1:36172 in state New
Sep 22 23:21:39.573 INFO [1] b937f86b-985c-4e08-8b98-07f7aae5f860 looper connected, looper: 1
Sep 22 23:21:39.573 INFO [1] Proc runs for 127.0.0.1:36532 in state New
Sep 22 23:21:39.573 INFO [2] b937f86b-985c-4e08-8b98-07f7aae5f860 looper connected, looper: 2
Sep 22 23:21:39.573 INFO [2] Proc runs for 127.0.0.1:33670 in state New
Sep 22 23:21:39.573 INFO Connection request from b937f86b-985c-4e08-8b98-07f7aae5f860 with version 4, task: proc
Sep 22 23:21:39.573 INFO upstairs UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:39.573 INFO Connection request from b937f86b-985c-4e08-8b98-07f7aae5f860 with version 4, task: proc
Sep 22 23:21:39.573 INFO upstairs UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:39.573 INFO Connection request from b937f86b-985c-4e08-8b98-07f7aae5f860 with version 4, task: proc
Sep 22 23:21:39.573 INFO upstairs UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:21:39.573 INFO [0] b937f86b-985c-4e08-8b98-07f7aae5f860 (27d81b06-bab5-4e4f-b4d3-4909cf4b2397) New New New ds_transition to WaitActive
Sep 22 23:21:39.573 INFO [0] Transition from New to WaitActive
Sep 22 23:21:39.573 INFO [1] b937f86b-985c-4e08-8b98-07f7aae5f860 (27d81b06-bab5-4e4f-b4d3-4909cf4b2397) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:39.573 INFO [1] Transition from New to WaitActive
Sep 22 23:21:39.574 INFO [2] b937f86b-985c-4e08-8b98-07f7aae5f860 (27d81b06-bab5-4e4f-b4d3-4909cf4b2397) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:39.574 INFO [2] Transition from New to WaitActive
Sep 22 23:21:39.574 INFO b937f86b-985c-4e08-8b98-07f7aae5f860 active request set
Sep 22 23:21:39.574 INFO [0] received activate with gen 1
Sep 22 23:21:39.574 INFO [0] client got ds_active_rx, promote! session 27d81b06-bab5-4e4f-b4d3-4909cf4b2397
Sep 22 23:21:39.574 INFO [1] received activate with gen 1
Sep 22 23:21:39.574 INFO [1] client got ds_active_rx, promote! session 27d81b06-bab5-4e4f-b4d3-4909cf4b2397
Sep 22 23:21:39.574 INFO [2] received activate with gen 1
Sep 22 23:21:39.574 INFO [2] client got ds_active_rx, promote! session 27d81b06-bab5-4e4f-b4d3-4909cf4b2397
Sep 22 23:21:39.574 INFO UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } is now active (read-write)
Sep 22 23:21:39.574 INFO UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } is now active (read-write)
Sep 22 23:21:39.575 INFO UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } is now active (read-write)
Sep 22 23:21:39.575 INFO [0] downstairs client at 127.0.0.1:36172 has UUID d1402af7-f301-4500-84d1-bcd26b1c483b
Sep 22 23:21:39.575 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: d1402af7-f301-4500-84d1-bcd26b1c483b, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:39.575 INFO b937f86b-985c-4e08-8b98-07f7aae5f860 WaitActive WaitActive WaitActive
Sep 22 23:21:39.575 INFO [1] downstairs client at 127.0.0.1:36532 has UUID e8aacc37-8406-4fc6-b948-447b29fd08c0
Sep 22 23:21:39.575 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: e8aacc37-8406-4fc6-b948-447b29fd08c0, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:39.575 INFO b937f86b-985c-4e08-8b98-07f7aae5f860 WaitActive WaitActive WaitActive
Sep 22 23:21:39.575 INFO [2] downstairs client at 127.0.0.1:33670 has UUID 213a7130-8159-4409-8d01-9d7ac9bbe737
Sep 22 23:21:39.575 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 213a7130-8159-4409-8d01-9d7ac9bbe737, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:39.575 INFO b937f86b-985c-4e08-8b98-07f7aae5f860 WaitActive WaitActive WaitActive
Sep 22 23:21:39.598 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.601 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.605 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.615 INFO UUID: 13d4e6fc-362c-415f-bc7d-909deb752536
Sep 22 23:21:39.615 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:39.615 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:39.615 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:39.615 INFO Using address: 127.0.0.1:52265, task: main
Sep 22 23:21:39.615 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:39.615 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:37101, task: repair
Sep 22 23:21:39.615 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:37101, task: repair
Sep 22 23:21:39.615 INFO listening, local_addr: 127.0.0.1:37101, task: repair
Sep 22 23:21:39.615 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:37101, task: repair
Sep 22 23:21:39.615 INFO Using repair address: 127.0.0.1:37101, task: main
Sep 22 23:21:39.615 INFO No SSL acceptor configured, task: main
Sep 22 23:21:39.636 INFO UUID: 9c7d4f0a-db95-4538-bb7e-7b1ed1936555
Sep 22 23:21:39.636 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:39.636 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:39.636 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:39.636 INFO Using address: 127.0.0.1:35718, task: main
Sep 22 23:21:39.636 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:39.636 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40347, task: repair
Sep 22 23:21:39.636 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40347, task: repair
Sep 22 23:21:39.636 INFO listening, local_addr: 127.0.0.1:40347, task: repair
Sep 22 23:21:39.636 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40347, task: repair
Sep 22 23:21:39.636 INFO Using repair address: 127.0.0.1:40347, task: main
Sep 22 23:21:39.636 INFO No SSL acceptor configured, task: main
Sep 22 23:21:39.656 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:39.658 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:39.660 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:39.660 INFO [0] b937f86b-985c-4e08-8b98-07f7aae5f860 (27d81b06-bab5-4e4f-b4d3-4909cf4b2397) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:39.660 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:39.660 WARN [0] new RM replaced this: None
Sep 22 23:21:39.660 INFO [0] Starts reconcile loop
Sep 22 23:21:39.661 INFO [1] b937f86b-985c-4e08-8b98-07f7aae5f860 (27d81b06-bab5-4e4f-b4d3-4909cf4b2397) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:39.661 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:39.661 WARN [1] new RM replaced this: None
Sep 22 23:21:39.661 INFO [1] Starts reconcile loop
Sep 22 23:21:39.661 INFO [2] b937f86b-985c-4e08-8b98-07f7aae5f860 (27d81b06-bab5-4e4f-b4d3-4909cf4b2397) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:39.661 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:39.661 WARN [2] new RM replaced this: None
Sep 22 23:21:39.661 INFO [2] Starts reconcile loop
Sep 22 23:21:39.661 INFO [0] 127.0.0.1:36172 task reports connection:true
Sep 22 23:21:39.661 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:39.661 INFO b937f86b-985c-4e08-8b98-07f7aae5f860 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:39.661 INFO [0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.661 INFO [0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.661 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:39.661 INFO [1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.661 INFO [1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.661 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:39.661 INFO [2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.661 INFO [2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.661 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:39.661 INFO Max found gen is 1
Sep 22 23:21:39.661 INFO Generation requested: 1 >= found:1
Sep 22 23:21:39.661 INFO Next flush: 1
Sep 22 23:21:39.661 INFO All extents match
Sep 22 23:21:39.661 INFO No downstairs repair required
Sep 22 23:21:39.661 INFO No initial repair work was required
Sep 22 23:21:39.661 INFO Set Downstairs and Upstairs active
Sep 22 23:21:39.661 INFO b937f86b-985c-4e08-8b98-07f7aae5f860 is now active with session: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397
Sep 22 23:21:39.661 INFO b937f86b-985c-4e08-8b98-07f7aae5f860 Set Active after no repair
Sep 22 23:21:39.661 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:39.661 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.661 INFO Opened existing region file "/tmp/downstairs-sK3WzHBh/region.json"
Sep 22 23:21:39.661 INFO Set check for repair
Sep 22 23:21:39.661 INFO Database read version 1
Sep 22 23:21:39.662 INFO Database write version 1
Sep 22 23:21:39.662 INFO [1] 127.0.0.1:36532 task reports connection:true
Sep 22 23:21:39.662 INFO b937f86b-985c-4e08-8b98-07f7aae5f860 Active Active Active
Sep 22 23:21:39.662 INFO Set check for repair
Sep 22 23:21:39.662 INFO [2] 127.0.0.1:33670 task reports connection:true
Sep 22 23:21:39.662 INFO b937f86b-985c-4e08-8b98-07f7aae5f860 Active Active Active
Sep 22 23:21:39.662 INFO Set check for repair
Sep 22 23:21:39.662 INFO [0] received reconcile message
Sep 22 23:21:39.662 INFO [0] All repairs completed, exit
Sep 22 23:21:39.662 INFO [0] Starts cmd_loop
Sep 22 23:21:39.662 INFO [1] received reconcile message
Sep 22 23:21:39.662 INFO [1] All repairs completed, exit
Sep 22 23:21:39.662 INFO [1] Starts cmd_loop
Sep 22 23:21:39.662 INFO [2] received reconcile message
Sep 22 23:21:39.662 INFO [2] All repairs completed, exit
Sep 22 23:21:39.662 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:39.673 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.673 INFO Created new region file "/tmp/downstairs-lohaNZJ5/region.json"
Sep 22 23:21:39.681 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.681 INFO Created new region file "/tmp/downstairs-Qt3syeMq/region.json"
Sep 22 23:21:39.723 WARN f712941d-0577-43e4-8a2e-7814ec270c09 request to replace downstairs 127.0.0.1:46295 with 127.0.0.1:47135
Sep 22 23:21:39.724 INFO f712941d-0577-43e4-8a2e-7814ec270c09 found new target: 127.0.0.1:47135 at 0
Sep 22 23:21:39.724 INFO Waiting for replacement to finish
Sep 22 23:21:39.729 INFO Checking if live repair is needed
Sep 22 23:21:39.729 INFO [0] f712941d-0577-43e4-8a2e-7814ec270c09 (512f09eb-7e91-421e-9fa9-f1bb0acbe6ae) LiveRepairReady Active Active ds_transition to LiveRepair
Sep 22 23:21:39.729 INFO [0] Transition from LiveRepairReady to LiveRepair
Sep 22 23:21:39.729 INFO Live Repair started
Sep 22 23:21:39.729 WARN Live Repair main task begins., task: repair
Sep 22 23:21:39.729 INFO Start Live Repair of extents 0 to 2, task: repair
Sep 22 23:21:39.729 INFO Start extent 0 repair, task: repair
Sep 22 23:21:39.729 DEBG RE:0 Repair extent begins
Sep 22 23:21:39.729 DEBG Create new job ids for 0, : downstairs
Sep 22 23:21:39.729 INFO RE:0 repair extent with ids 1002,1003,1004,1005 deps:[]
Sep 22 23:21:39.729 DEBG Enqueue repair job 1005, : downstairs
Sep 22 23:21:39.729 DEBG Enqueue repair job 1002, : downstairs
Sep 22 23:21:39.730 INFO RE:0 close id:1002 queued, notify DS
Sep 22 23:21:39.730 INFO RE:0 Wait for result from close command 1002:3
Sep 22 23:21:39.730 DEBG [0] 1002 Remove check skipped:{JobId(1001)} from deps:[], : downstairs
Sep 22 23:21:39.730 DEBG [0] 1002 Remove check < min repaired:1002 from deps:[], : downstairs
Sep 22 23:21:39.730 INFO [0] 1002 final dependency list [], : downstairs
Sep 22 23:21:39.730 DEBG [0] 1005 Remove check skipped:{JobId(1001)} from deps:[JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:21:39.730 DEBG [0] 1005 Remove check < min repaired:1002 from deps:[JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:21:39.730 INFO [0] 1005 final dependency list [JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:21:39.731 DEBG Flush just extent 0 with f:2 and g:1
Sep 22 23:21:39.732 DEBG FlushClose:1002 extent 0 deps:[] res:true f:2 g:1
Sep 22 23:21:39.732 DEBG Flush just extent 0 with f:2 and g:1
Sep 22 23:21:39.733 DEBG FlushClose:1002 extent 0 deps:[] res:true f:2 g:1
Sep 22 23:21:39.734 DEBG JustClose :1002 extent 0 deps:[] res:true
Sep 22 23:21:39.734 DEBG [1] ELC got g:1 f:1 d:false
Sep 22 23:21:39.734 DEBG [2] ELC got g:1 f:1 d:false
Sep 22 23:21:39.734 DEBG [0] ELC got g:0 f:0 d:false
Sep 22 23:21:39.734 DEBG [0] ExtentFlushClose 1002 AckReady, : downstairs
Sep 22 23:21:39.734 DEBG up_ds_listen was notified
Sep 22 23:21:39.734 DEBG up_ds_listen process 1002
Sep 22 23:21:39.734 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:39.734 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:39.734 DEBG Extent 0 id:1002 Done
Sep 22 23:21:39.735 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:21:39.735 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:21:39.735 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
Sep 22 23:21:39.735 DEBG Enqueue repair job 1003, : downstairs
Sep 22 23:21:39.735 INFO RE:0 Wait for result from repair command 1003:4
Sep 22 23:21:39.735 DEBG [0] 1003 Remove check skipped:{JobId(1001)} from deps:[JobId(1002)], : downstairs
Sep 22 23:21:39.735 DEBG [0] 1003 Remove check < min repaired:1002 from deps:[JobId(1002)], : downstairs
Sep 22 23:21:39.735 INFO [0] 1003 final dependency list [JobId(1002)], : downstairs
Sep 22 23:21:39.735 DEBG Received NoOP 1003
Sep 22 23:21:39.735 DEBG Received NoOP 1003
Sep 22 23:21:39.735 DEBG Received ExtentLiveRepair 1003
Sep 22 23:21:39.735 DEBG Work of: LiveNoOp 1003
Sep 22 23:21:39.735 DEBG LiveNoOp  :1003 deps:[JobId(1002)] res:true
Sep 22 23:21:39.735 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:39.735 DEBG Work of: LiveNoOp 1003
Sep 22 23:21:39.735 DEBG LiveNoOp  :1003 deps:[JobId(1002)] res:true
Sep 22 23:21:39.736 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:39.736 DEBG ExtentLiveRepair: extent 0 sra:127.0.0.1:39249
Sep 22 23:21:39.736 INFO Created copy dir "/tmp/downstairs-qImf5Xje/00/000/000.copy"
Sep 22 23:21:39.775 INFO UUID: 99fe08ae-482d-4baf-90c5-1e5bdfb86144
Sep 22 23:21:39.775 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:39.775 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:39.775 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:39.775 INFO Using address: 127.0.0.1:33021, task: main
Sep 22 23:21:39.776 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:39.776 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46198, task: repair
Sep 22 23:21:39.776 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46198, task: repair
Sep 22 23:21:39.776 INFO listening, local_addr: 127.0.0.1:46198, task: repair
Sep 22 23:21:39.776 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46198, task: repair
Sep 22 23:21:39.776 INFO Using repair address: 127.0.0.1:46198, task: main
Sep 22 23:21:39.776 INFO No SSL acceptor configured, task: main
Sep 22 23:21:39.822 INFO accepted connection, remote_addr: 127.0.0.1:62976, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.823 TRCE incoming request, uri: /extent/0/files, method: GET, req_id: 1d365637-bb93-4b60-850e-025d1c909172, remote_addr: 127.0.0.1:62976, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.823 INFO request completed, latency_us: 339, response_code: 200, uri: /extent/0/files, method: GET, req_id: 1d365637-bb93-4b60-850e-025d1c909172, remote_addr: 127.0.0.1:62976, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.824 INFO eid:0 Found repair files: ["000", "000.db"]
Sep 22 23:21:39.824 TRCE incoming request, uri: /newextent/0/data, method: GET, req_id: b54257c3-7bf3-420e-b2ca-082dc62e4f6a, remote_addr: 127.0.0.1:62976, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.824 INFO request completed, latency_us: 494, response_code: 200, uri: /newextent/0/data, method: GET, req_id: b54257c3-7bf3-420e-b2ca-082dc62e4f6a, remote_addr: 127.0.0.1:62976, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.825 TRCE incoming request, uri: /newextent/0/db, method: GET, req_id: 9ca46789-c153-401c-aaa3-28385442d849, remote_addr: 127.0.0.1:62976, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.826 INFO request completed, latency_us: 326, response_code: 200, uri: /newextent/0/db, method: GET, req_id: 9ca46789-c153-401c-aaa3-28385442d849, remote_addr: 127.0.0.1:62976, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.826 INFO Repair files downloaded, move directory "/tmp/downstairs-qImf5Xje/00/000/000.copy" to "/tmp/downstairs-qImf5Xje/00/000/000.replace"
Sep 22 23:21:39.827 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000"
Sep 22 23:21:39.828 INFO Copy files from "/tmp/downstairs-qImf5Xje/00/000/000.replace" in "/tmp/downstairs-qImf5Xje/00/000"
Sep 22 23:21:39.828 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000/000"
Sep 22 23:21:39.828 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000/000.db"
Sep 22 23:21:39.828 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000"
Sep 22 23:21:39.828 INFO Move directory  "/tmp/downstairs-qImf5Xje/00/000/000.replace" to "/tmp/downstairs-qImf5Xje/00/000/000.completed"
Sep 22 23:21:39.828 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000"
Sep 22 23:21:39.828 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000"
Sep 22 23:21:39.828 DEBG LiveRepair:1003 extent 0 deps:[JobId(1002)] res:true
Sep 22 23:21:39.828 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:39.828 DEBG [0] ExtentLiveRepair AckReady 1003, : downstairs
Sep 22 23:21:39.828 DEBG up_ds_listen was notified
Sep 22 23:21:39.828 DEBG up_ds_listen process 1003
Sep 22 23:21:39.828 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:39.829 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:39.829 DEBG Extent 0 id:1003 Done
Sep 22 23:21:39.829 DEBG Enqueue repair job 1004, : downstairs
Sep 22 23:21:39.829 INFO RE:0 Wait for result from NoOp command 1004:5
Sep 22 23:21:39.829 DEBG [0] 1004 Remove check skipped:{JobId(1001)} from deps:[JobId(1002), JobId(1003)], : downstairs
Sep 22 23:21:39.829 DEBG [0] 1004 Remove check < min repaired:1002 from deps:[JobId(1002), JobId(1003)], : downstairs
Sep 22 23:21:39.829 INFO [0] 1004 final dependency list [JobId(1002), JobId(1003)], : downstairs
Sep 22 23:21:39.829 DEBG Received NoOP 1004
Sep 22 23:21:39.829 DEBG Received NoOP 1004
Sep 22 23:21:39.829 DEBG Received NoOP 1004
Sep 22 23:21:39.829 DEBG Work of: LiveNoOp 1004
Sep 22 23:21:39.829 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:21:39.830 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:21:39.830 DEBG Work of: LiveNoOp 1004
Sep 22 23:21:39.830 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:21:39.831 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:21:39.832 DEBG Work of: LiveNoOp 1004
Sep 22 23:21:39.832 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:21:39.833 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:21:39.833 DEBG [0] ExtentLiveNoOp AckReady 1004, : downstairs
Sep 22 23:21:39.833 DEBG [0] ExtentLiveReopen AckReady 1005, : downstairs
Sep 22 23:21:39.833 DEBG up_ds_listen was notified
Sep 22 23:21:39.833 DEBG up_ds_listen process 1004
Sep 22 23:21:39.833 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:21:39.833 DEBG up_ds_listen process 1005
Sep 22 23:21:39.833 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:21:39.834 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:21:39.834 DEBG up_ds_listen was notified
Sep 22 23:21:39.834 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:21:39.834 DEBG Extent 0 id:1004 Done
Sep 22 23:21:39.834 INFO RE:0 Wait for result from reopen command 1005:6
Sep 22 23:21:39.834 DEBG Extent 0 id:1005 Done
Sep 22 23:21:39.834 INFO Start extent 1 repair, task: repair
Sep 22 23:21:39.834 DEBG RE:1 Repair extent begins
Sep 22 23:21:39.834 DEBG Create new job ids for 1, : downstairs
Sep 22 23:21:39.834 INFO RE:1 repair extent with ids 1006,1007,1008,1009 deps:[]
Sep 22 23:21:39.834 DEBG Enqueue repair job 1009, : downstairs
Sep 22 23:21:39.834 DEBG Enqueue repair job 1006, : downstairs
Sep 22 23:21:39.834 INFO RE:1 close id:1006 queued, notify DS
Sep 22 23:21:39.834 INFO RE:1 Wait for result from close command 1006:7
Sep 22 23:21:39.834 DEBG [0] 1006 Remove check skipped:{JobId(1001)} from deps:[], : downstairs
Sep 22 23:21:39.834 DEBG [0] 1006 Remove check < min repaired:1002 from deps:[], : downstairs
Sep 22 23:21:39.834 INFO [0] 1006 final dependency list [], : downstairs
Sep 22 23:21:39.834 DEBG [0] 1009 Remove check skipped:{JobId(1001)} from deps:[JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:21:39.834 DEBG [0] 1009 Remove check < min repaired:1002 from deps:[JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:21:39.834 INFO [0] 1009 final dependency list [JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:21:39.834 DEBG Flush just extent 1 with f:3 and g:1
Sep 22 23:21:39.835 DEBG FlushClose:1006 extent 1 deps:[] res:true f:3 g:1
Sep 22 23:21:39.835 DEBG Flush just extent 1 with f:3 and g:1
Sep 22 23:21:39.836 DEBG FlushClose:1006 extent 1 deps:[] res:true f:3 g:1
Sep 22 23:21:39.837 DEBG JustClose :1006 extent 1 deps:[] res:true
Sep 22 23:21:39.838 DEBG [1] ELC got g:1 f:1 d:false
Sep 22 23:21:39.838 DEBG [2] ELC got g:1 f:1 d:false
Sep 22 23:21:39.838 DEBG [0] ELC got g:0 f:0 d:false
Sep 22 23:21:39.838 DEBG [0] ExtentFlushClose 1006 AckReady, : downstairs
Sep 22 23:21:39.838 DEBG up_ds_listen was notified
Sep 22 23:21:39.838 DEBG up_ds_listen process 1006
Sep 22 23:21:39.838 DEBG [A] ack job 1006:7, : downstairs
Sep 22 23:21:39.838 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:39.838 DEBG Extent 1 id:1006 Done
Sep 22 23:21:39.838 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:21:39.838 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:21:39.838 INFO Repair for extent 1 s:2 d:[ClientId(0)], : downstairs
Sep 22 23:21:39.838 DEBG Enqueue repair job 1007, : downstairs
Sep 22 23:21:39.838 INFO RE:1 Wait for result from repair command 1007:8
Sep 22 23:21:39.838 DEBG [0] 1007 Remove check skipped:{JobId(1001)} from deps:[JobId(1006)], : downstairs
Sep 22 23:21:39.838 DEBG [0] 1007 Remove check < min repaired:1002 from deps:[JobId(1006)], : downstairs
Sep 22 23:21:39.838 INFO [0] 1007 final dependency list [JobId(1006)], : downstairs
Sep 22 23:21:39.838 DEBG Received NoOP 1007
Sep 22 23:21:39.838 DEBG Received NoOP 1007
Sep 22 23:21:39.838 DEBG Received ExtentLiveRepair 1007
Sep 22 23:21:39.839 DEBG Work of: LiveNoOp 1007
Sep 22 23:21:39.839 DEBG LiveNoOp  :1007 deps:[JobId(1006)] res:true
Sep 22 23:21:39.839 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:39.839 DEBG Work of: LiveNoOp 1007
Sep 22 23:21:39.839 DEBG LiveNoOp  :1007 deps:[JobId(1006)] res:true
Sep 22 23:21:39.839 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:39.839 DEBG ExtentLiveRepair: extent 1 sra:127.0.0.1:39249
Sep 22 23:21:39.839 INFO Created copy dir "/tmp/downstairs-qImf5Xje/00/000/001.copy"
Sep 22 23:21:39.851 INFO Upstairs starts
Sep 22 23:21:39.851 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:39.851 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:39.851 INFO Crucible stats registered with UUID: 9144e02c-c312-47c4-9b1c-f03618834608
Sep 22 23:21:39.851 INFO Crucible 9144e02c-c312-47c4-9b1c-f03618834608 has session id: 20abe3f6-9684-44cd-ad20-faf7fc8fd696
Sep 22 23:21:39.851 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:39.851 INFO [0] connecting to 127.0.0.1:52165, looper: 0
Sep 22 23:21:39.851 INFO UUID: efa5c554-210a-4196-815d-fa3d1dfbb858
Sep 22 23:21:39.851 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:39.852 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:39.852 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:39.852 INFO Using address: 127.0.0.1:37034, task: main
Sep 22 23:21:39.852 INFO [1] connecting to 127.0.0.1:48339, looper: 1
Sep 22 23:21:39.852 INFO [2] connecting to 127.0.0.1:33021, looper: 2
Sep 22 23:21:39.852 INFO up_listen starts, task: up_listen
Sep 22 23:21:39.852 INFO Wait for all three downstairs to come online
Sep 22 23:21:39.852 INFO Flush timeout: 0.5
Sep 22 23:21:39.852 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:39.852 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:59864, task: repair
Sep 22 23:21:39.852 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:59864, task: repair
Sep 22 23:21:39.852 INFO listening, local_addr: 127.0.0.1:59864, task: repair
Sep 22 23:21:39.852 INFO accepted connection from 127.0.0.1:33221, task: main
Sep 22 23:21:39.852 INFO accepted connection from 127.0.0.1:35948, task: main
Sep 22 23:21:39.852 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:59864, task: repair
Sep 22 23:21:39.852 INFO Using repair address: 127.0.0.1:59864, task: main
Sep 22 23:21:39.852 INFO No SSL acceptor configured, task: main
Sep 22 23:21:39.852 INFO accepted connection from 127.0.0.1:45718, task: main
Sep 22 23:21:39.852 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 looper connected, looper: 0
Sep 22 23:21:39.852 INFO [0] Proc runs for 127.0.0.1:52165 in state New
Sep 22 23:21:39.852 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 looper connected, looper: 1
Sep 22 23:21:39.853 INFO [1] Proc runs for 127.0.0.1:48339 in state New
Sep 22 23:21:39.853 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 looper connected, looper: 2
Sep 22 23:21:39.853 INFO [2] Proc runs for 127.0.0.1:33021 in state New
Sep 22 23:21:39.853 INFO Connection request from 9144e02c-c312-47c4-9b1c-f03618834608 with version 4, task: proc
Sep 22 23:21:39.853 INFO upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:39.853 INFO Connection request from 9144e02c-c312-47c4-9b1c-f03618834608 with version 4, task: proc
Sep 22 23:21:39.853 INFO upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:39.853 INFO Connection request from 9144e02c-c312-47c4-9b1c-f03618834608 with version 4, task: proc
Sep 22 23:21:39.853 INFO upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:21:39.854 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) New New New ds_transition to WaitActive
Sep 22 23:21:39.854 INFO [0] Transition from New to WaitActive
Sep 22 23:21:39.854 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:39.854 INFO [1] Transition from New to WaitActive
Sep 22 23:21:39.854 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:39.854 INFO [2] Transition from New to WaitActive
Sep 22 23:21:39.854 INFO 9144e02c-c312-47c4-9b1c-f03618834608 active request set
Sep 22 23:21:39.854 INFO [0] received activate with gen 1
Sep 22 23:21:39.854 INFO [0] client got ds_active_rx, promote! session cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f
Sep 22 23:21:39.854 INFO [1] received activate with gen 1
Sep 22 23:21:39.854 INFO [1] client got ds_active_rx, promote! session cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f
Sep 22 23:21:39.854 INFO [2] received activate with gen 1
Sep 22 23:21:39.854 INFO [2] client got ds_active_rx, promote! session cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f
Sep 22 23:21:39.855 INFO UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } is now active (read-write)
Sep 22 23:21:39.855 INFO UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } is now active (read-write)
Sep 22 23:21:39.855 INFO UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } is now active (read-write)
Sep 22 23:21:39.856 INFO [0] downstairs client at 127.0.0.1:52165 has UUID 63b48fd3-ffb2-4c20-bff7-4a64980fdfc8
Sep 22 23:21:39.856 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 63b48fd3-ffb2-4c20-bff7-4a64980fdfc8, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:39.856 INFO 9144e02c-c312-47c4-9b1c-f03618834608 WaitActive WaitActive WaitActive
Sep 22 23:21:39.856 INFO [1] downstairs client at 127.0.0.1:48339 has UUID 0b1643bf-b244-4616-9246-4922ec6009af
Sep 22 23:21:39.856 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 0b1643bf-b244-4616-9246-4922ec6009af, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:39.856 INFO 9144e02c-c312-47c4-9b1c-f03618834608 WaitActive WaitActive WaitActive
Sep 22 23:21:39.856 INFO [2] downstairs client at 127.0.0.1:33021 has UUID 99fe08ae-482d-4baf-90c5-1e5bdfb86144
Sep 22 23:21:39.856 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 99fe08ae-482d-4baf-90c5-1e5bdfb86144, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:39.856 INFO 9144e02c-c312-47c4-9b1c-f03618834608 WaitActive WaitActive WaitActive
Sep 22 23:21:39.916 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.916 INFO Created new region file "/tmp/downstairs-6BO8rCTH/region.json"
Sep 22 23:21:39.917 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.919 INFO accepted connection, remote_addr: 127.0.0.1:60896, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.919 TRCE incoming request, uri: /extent/1/files, method: GET, req_id: c23f1323-c074-47d8-a9d1-bb3a0f448154, remote_addr: 127.0.0.1:60896, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.919 INFO request completed, latency_us: 321, response_code: 200, uri: /extent/1/files, method: GET, req_id: c23f1323-c074-47d8-a9d1-bb3a0f448154, remote_addr: 127.0.0.1:60896, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.920 INFO eid:1 Found repair files: ["001", "001.db"]
Sep 22 23:21:39.920 TRCE incoming request, uri: /newextent/1/data, method: GET, req_id: b118c6b2-4bdf-44de-905e-b7463c90509c, remote_addr: 127.0.0.1:60896, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.921 INFO request completed, latency_us: 419, response_code: 200, uri: /newextent/1/data, method: GET, req_id: b118c6b2-4bdf-44de-905e-b7463c90509c, remote_addr: 127.0.0.1:60896, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.922 TRCE incoming request, uri: /newextent/1/db, method: GET, req_id: 5e669d1a-a88e-41a6-ba42-2171b56bd167, remote_addr: 127.0.0.1:60896, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.922 INFO request completed, latency_us: 342, response_code: 200, uri: /newextent/1/db, method: GET, req_id: 5e669d1a-a88e-41a6-ba42-2171b56bd167, remote_addr: 127.0.0.1:60896, local_addr: 127.0.0.1:39249, task: repair
Sep 22 23:21:39.923 INFO Repair files downloaded, move directory "/tmp/downstairs-qImf5Xje/00/000/001.copy" to "/tmp/downstairs-qImf5Xje/00/000/001.replace"
Sep 22 23:21:39.923 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000"
Sep 22 23:21:39.924 INFO Copy files from "/tmp/downstairs-qImf5Xje/00/000/001.replace" in "/tmp/downstairs-qImf5Xje/00/000"
Sep 22 23:21:39.924 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000/001"
Sep 22 23:21:39.924 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000/001.db"
Sep 22 23:21:39.924 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000"
Sep 22 23:21:39.924 INFO Move directory  "/tmp/downstairs-qImf5Xje/00/000/001.replace" to "/tmp/downstairs-qImf5Xje/00/000/001.completed"
Sep 22 23:21:39.924 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000"
Sep 22 23:21:39.924 DEBG fsync completed for: "/tmp/downstairs-qImf5Xje/00/000"
Sep 22 23:21:39.924 DEBG LiveRepair:1007 extent 1 deps:[JobId(1006)] res:true
Sep 22 23:21:39.925 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: f712941d-0577-43e4-8a2e-7814ec270c09, session_id: 512f09eb-7e91-421e-9fa9-f1bb0acbe6ae, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:39.925 DEBG [0] ExtentLiveRepair AckReady 1007, : downstairs
Sep 22 23:21:39.925 DEBG up_ds_listen was notified
Sep 22 23:21:39.925 DEBG up_ds_listen process 1007
Sep 22 23:21:39.925 DEBG [A] ack job 1007:8, : downstairs
Sep 22 23:21:39.925 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:39.925 DEBG Extent 1 id:1007 Done
Sep 22 23:21:39.925 DEBG Enqueue repair job 1008, : downstairs
Sep 22 23:21:39.925 INFO RE:1 Wait for result from NoOp command 1008:9
Sep 22 23:21:39.925 DEBG [0] 1008 Remove check skipped:{JobId(1001)} from deps:[JobId(1006), JobId(1007)], : downstairs
Sep 22 23:21:39.925 DEBG [0] 1008 Remove check < min repaired:1002 from deps:[JobId(1006), JobId(1007)], : downstairs
Sep 22 23:21:39.925 INFO [0] 1008 final dependency list [JobId(1006), JobId(1007)], : downstairs
Sep 22 23:21:39.926 DEBG Received NoOP 1008
Sep 22 23:21:39.926 DEBG Received NoOP 1008
Sep 22 23:21:39.926 DEBG Received NoOP 1008
Sep 22 23:21:39.926 DEBG Work of: LiveNoOp 1008
Sep 22 23:21:39.926 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:21:39.927 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:21:39.927 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:39.927 DEBG Work of: LiveNoOp 1008
Sep 22 23:21:39.927 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:21:39.927 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.927 INFO Opened existing region file "/tmp/downstairs-lohaNZJ5/region.json"
Sep 22 23:21:39.927 INFO Database read version 1
Sep 22 23:21:39.927 INFO Database write version 1
Sep 22 23:21:39.929 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:21:39.929 DEBG Work of: LiveNoOp 1008
Sep 22 23:21:39.929 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:21:39.930 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:39.930 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:21:39.931 DEBG [0] ExtentLiveNoOp AckReady 1008, : downstairs
Sep 22 23:21:39.931 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:39.931 INFO Opened existing region file "/tmp/downstairs-Qt3syeMq/region.json"
Sep 22 23:21:39.931 DEBG [0] ExtentLiveReopen AckReady 1009, : downstairs
Sep 22 23:21:39.931 INFO Database read version 1
Sep 22 23:21:39.931 INFO Database write version 1
Sep 22 23:21:39.931 DEBG up_ds_listen was notified
Sep 22 23:21:39.931 DEBG up_ds_listen process 1008
Sep 22 23:21:39.931 DEBG [A] ack job 1008:9, : downstairs
Sep 22 23:21:39.931 DEBG up_ds_listen process 1009
Sep 22 23:21:39.931 DEBG [A] ack job 1009:10, : downstairs
Sep 22 23:21:39.931 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:21:39.931 DEBG up_ds_listen was notified
Sep 22 23:21:39.931 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:21:39.931 DEBG Extent 1 id:1008 Done
Sep 22 23:21:39.931 INFO RE:1 Wait for result from reopen command 1009:10
Sep 22 23:21:39.931 DEBG Extent 1 id:1009 Done
Sep 22 23:21:39.931 DEBG IO Flush 1010 has deps [JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)]
Sep 22 23:21:39.931 INFO LiveRepair final flush submitted
Sep 22 23:21:39.931 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.931 DEBG [0] 1010 Remove check skipped:{JobId(1001)} from deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:21:39.931 DEBG [0] 1010 Remove check < min repaired:1002 from deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:21:39.931 INFO [0] 1010 final dependency list [JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:21:39.932 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:21:39.932 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:21:39.932 DEBG Flush     :1010 extent_limit Some(1) deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:21:39.932 DEBG up_ds_listen was notified
Sep 22 23:21:39.933 DEBG up_ds_listen process 1010
Sep 22 23:21:39.933 DEBG [A] ack job 1010:11, : downstairs
Sep 22 23:21:39.933 DEBG [rc] retire 1010 clears [JobId(1002), JobId(1003), JobId(1004), JobId(1005), JobId(1006), JobId(1007), JobId(1008), JobId(1009), JobId(1010)], : downstairs
Sep 22 23:21:39.933 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:39.933 INFO LiveRepair final flush completed
Sep 22 23:21:39.933 INFO [0] f712941d-0577-43e4-8a2e-7814ec270c09 (512f09eb-7e91-421e-9fa9-f1bb0acbe6ae) LiveRepair Active Active ds_transition to Active
Sep 22 23:21:39.933 INFO [0] Transition from LiveRepair to Active
Sep 22 23:21:39.933 WARN Live Repair returns Ok(())
Sep 22 23:21:39.935 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:39.952 INFO UUID: dfb16c90-27d7-4a45-bd56-019dbea3b0bb
Sep 22 23:21:39.952 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:39.952 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:39.952 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:39.952 INFO Using address: 127.0.0.1:65451, task: main
Sep 22 23:21:39.952 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:39.953 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56310, task: repair
Sep 22 23:21:39.953 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56310, task: repair
Sep 22 23:21:39.953 INFO listening, local_addr: 127.0.0.1:56310, task: repair
Sep 22 23:21:39.953 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56310, task: repair
Sep 22 23:21:39.953 INFO Using repair address: 127.0.0.1:56310, task: main
Sep 22 23:21:39.953 INFO No SSL acceptor configured, task: main
Sep 22 23:21:39.990 WARN 52fe8d40-333d-4b64-8663-2adf0476947f request to replace downstairs 127.0.0.1:38958 with 127.0.0.1:62901
Sep 22 23:21:39.990 INFO 52fe8d40-333d-4b64-8663-2adf0476947f found new target: 127.0.0.1:62901 at 0
Waiting for replacement to finish
Sep 22 23:21:39.995 INFO Checking if live repair is needed
Sep 22 23:21:39.995 INFO [0] 52fe8d40-333d-4b64-8663-2adf0476947f (fc0a5f14-ae91-4f48-aded-c2a3fecef8a0) LiveRepairReady Active Active ds_transition to LiveRepair
Sep 22 23:21:39.995 INFO [0] Transition from LiveRepairReady to LiveRepair
Sep 22 23:21:39.995 INFO Live Repair started
Sep 22 23:21:39.995 WARN Live Repair main task begins., task: repair
Sep 22 23:21:39.995 INFO Start Live Repair of extents 0 to 2, task: repair
Sep 22 23:21:39.995 INFO Start extent 0 repair, task: repair
Sep 22 23:21:39.995 DEBG RE:0 Repair extent begins
Sep 22 23:21:39.995 DEBG Create new job ids for 0, : downstairs
Sep 22 23:21:39.995 INFO RE:0 repair extent with ids 1002,1003,1004,1005 deps:[]
Sep 22 23:21:39.996 DEBG Enqueue repair job 1005, : downstairs
Sep 22 23:21:39.996 DEBG Enqueue repair job 1002, : downstairs
Sep 22 23:21:39.996 INFO RE:0 close id:1002 queued, notify DS
Sep 22 23:21:39.996 INFO RE:0 Wait for result from close command 1002:3
Sep 22 23:21:39.996 DEBG [0] 1002 Remove check skipped:{JobId(1001)} from deps:[], : downstairs
Sep 22 23:21:39.996 DEBG [0] 1002 Remove check < min repaired:1002 from deps:[], : downstairs
Sep 22 23:21:39.996 INFO [0] 1002 final dependency list [], : downstairs
Sep 22 23:21:39.996 DEBG [0] 1005 Remove check skipped:{JobId(1001)} from deps:[JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:21:39.996 DEBG [0] 1005 Remove check < min repaired:1002 from deps:[JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:21:39.996 INFO [0] 1005 final dependency list [JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:21:39.996 DEBG Flush just extent 0 with f:2 and g:1
Sep 22 23:21:39.997 DEBG FlushClose:1002 extent 0 deps:[] res:true f:2 g:1
Sep 22 23:21:39.997 DEBG Flush just extent 0 with f:2 and g:1
Sep 22 23:21:39.998 DEBG FlushClose:1002 extent 0 deps:[] res:true f:2 g:1
Sep 22 23:21:39.999 DEBG JustClose :1002 extent 0 deps:[] res:true
Sep 22 23:21:39.999 DEBG [1] ELC got g:1 f:1 d:false
Sep 22 23:21:39.999 DEBG [2] ELC got g:1 f:1 d:false
Sep 22 23:21:39.999 DEBG [0] ELC got g:0 f:0 d:false
Sep 22 23:21:39.999 DEBG [0] ExtentFlushClose 1002 AckReady, : downstairs
Sep 22 23:21:39.999 DEBG up_ds_listen was notified
Sep 22 23:21:39.999 DEBG up_ds_listen process 1002
Sep 22 23:21:39.999 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:39.999 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:39.999 DEBG Extent 0 id:1002 Done
Sep 22 23:21:39.999 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:21:39.999 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:21:39.999 INFO Repair for extent 0 s:2 d:[ClientId(0)], : downstairs
Sep 22 23:21:39.999 DEBG Enqueue repair job 1003, : downstairs
Sep 22 23:21:39.999 INFO RE:0 Wait for result from repair command 1003:4
Sep 22 23:21:40.000 DEBG [0] 1003 Remove check skipped:{JobId(1001)} from deps:[JobId(1002)], : downstairs
Sep 22 23:21:40.000 DEBG [0] 1003 Remove check < min repaired:1002 from deps:[JobId(1002)], : downstairs
Sep 22 23:21:40.000 INFO [0] 1003 final dependency list [JobId(1002)], : downstairs
Sep 22 23:21:40.000 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:40.000 DEBG Received NoOP 1003
Sep 22 23:21:40.000 DEBG Received NoOP 1003
Sep 22 23:21:40.000 INFO Created new region file "/tmp/downstairs-AuFNRJc7/region.json"
Sep 22 23:21:40.000 DEBG Received ExtentLiveRepair 1003
Sep 22 23:21:40.000 DEBG Work of: LiveNoOp 1003
Sep 22 23:21:40.000 DEBG LiveNoOp  :1003 deps:[JobId(1002)] res:true
Sep 22 23:21:40.000 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:40.000 DEBG Work of: LiveNoOp 1003
Sep 22 23:21:40.000 DEBG LiveNoOp  :1003 deps:[JobId(1002)] res:true
Sep 22 23:21:40.000 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:40.001 DEBG ExtentLiveRepair: extent 0 sra:127.0.0.1:53407
Sep 22 23:21:40.001 INFO Created copy dir "/tmp/downstairs-kiLstbeI/00/000/000.copy"
Sep 22 23:21:40.014 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.019 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.023 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.024 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:40.024 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:40.024 WARN [0] new RM replaced this: None
Sep 22 23:21:40.024 INFO [0] Starts reconcile loop
Sep 22 23:21:40.024 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:40.024 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:40.024 WARN [1] new RM replaced this: None
Sep 22 23:21:40.024 INFO [1] Starts reconcile loop
Sep 22 23:21:40.025 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:40.025 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:40.025 WARN [2] new RM replaced this: None
Sep 22 23:21:40.025 INFO [2] Starts reconcile loop
Sep 22 23:21:40.025 INFO [0] 127.0.0.1:52165 task reports connection:true
Sep 22 23:21:40.025 INFO 9144e02c-c312-47c4-9b1c-f03618834608 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:40.025 INFO [0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.025 INFO [0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.025 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:40.025 INFO [1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.025 INFO [1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.025 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:40.025 INFO [2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.025 INFO [2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.025 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:40.025 INFO Max found gen is 1
Sep 22 23:21:40.025 INFO Generation requested: 1 >= found:1
Sep 22 23:21:40.025 INFO Next flush: 1
Sep 22 23:21:40.025 INFO All extents match
Sep 22 23:21:40.025 INFO No downstairs repair required
Sep 22 23:21:40.025 INFO No initial repair work was required
Sep 22 23:21:40.025 INFO Set Downstairs and Upstairs active
Sep 22 23:21:40.025 INFO 9144e02c-c312-47c4-9b1c-f03618834608 is now active with session: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f
Sep 22 23:21:40.025 INFO 9144e02c-c312-47c4-9b1c-f03618834608 Set Active after no repair
Sep 22 23:21:40.025 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:40.026 INFO Set check for repair
Sep 22 23:21:40.026 INFO [1] 127.0.0.1:48339 task reports connection:true
Sep 22 23:21:40.026 INFO 9144e02c-c312-47c4-9b1c-f03618834608 Active Active Active
Sep 22 23:21:40.026 INFO Set check for repair
Sep 22 23:21:40.026 INFO [2] 127.0.0.1:33021 task reports connection:true
Sep 22 23:21:40.026 INFO 9144e02c-c312-47c4-9b1c-f03618834608 Active Active Active
Sep 22 23:21:40.026 INFO Set check for repair
Sep 22 23:21:40.026 INFO [0] received reconcile message
Sep 22 23:21:40.026 INFO [0] All repairs completed, exit
Sep 22 23:21:40.026 INFO [0] Starts cmd_loop
Sep 22 23:21:40.026 INFO [1] received reconcile message
Sep 22 23:21:40.026 INFO [1] All repairs completed, exit
Sep 22 23:21:40.026 INFO [1] Starts cmd_loop
Sep 22 23:21:40.026 INFO [2] received reconcile message
Sep 22 23:21:40.026 INFO [2] All repairs completed, exit
Sep 22 23:21:40.026 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:40.102 INFO accepted connection, remote_addr: 127.0.0.1:36805, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.103 TRCE incoming request, uri: /extent/0/files, method: GET, req_id: fe714542-aed6-499e-b3c9-47b0afd62646, remote_addr: 127.0.0.1:36805, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.103 INFO request completed, latency_us: 348, response_code: 200, uri: /extent/0/files, method: GET, req_id: fe714542-aed6-499e-b3c9-47b0afd62646, remote_addr: 127.0.0.1:36805, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.103 INFO eid:0 Found repair files: ["000", "000.db"]
Sep 22 23:21:40.104 TRCE incoming request, uri: /newextent/0/data, method: GET, req_id: 799cab3d-ab6f-4bed-a0be-5df21ab7e965, remote_addr: 127.0.0.1:36805, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.104 INFO request completed, latency_us: 554, response_code: 200, uri: /newextent/0/data, method: GET, req_id: 799cab3d-ab6f-4bed-a0be-5df21ab7e965, remote_addr: 127.0.0.1:36805, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.105 TRCE incoming request, uri: /newextent/0/db, method: GET, req_id: 7fb1c097-e8a3-45ec-908b-3041a969cd7c, remote_addr: 127.0.0.1:36805, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.106 INFO request completed, latency_us: 397, response_code: 200, uri: /newextent/0/db, method: GET, req_id: 7fb1c097-e8a3-45ec-908b-3041a969cd7c, remote_addr: 127.0.0.1:36805, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.107 INFO Repair files downloaded, move directory "/tmp/downstairs-kiLstbeI/00/000/000.copy" to "/tmp/downstairs-kiLstbeI/00/000/000.replace"
Sep 22 23:21:40.107 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000"
Sep 22 23:21:40.108 INFO Copy files from "/tmp/downstairs-kiLstbeI/00/000/000.replace" in "/tmp/downstairs-kiLstbeI/00/000"
Sep 22 23:21:40.108 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000/000"
Sep 22 23:21:40.108 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000/000.db"
Sep 22 23:21:40.108 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000"
Sep 22 23:21:40.108 INFO Move directory  "/tmp/downstairs-kiLstbeI/00/000/000.replace" to "/tmp/downstairs-kiLstbeI/00/000/000.completed"
Sep 22 23:21:40.108 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000"
Sep 22 23:21:40.108 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000"
Sep 22 23:21:40.108 DEBG LiveRepair:1003 extent 0 deps:[JobId(1002)] res:true
Sep 22 23:21:40.108 WARN 1005 job ELiveReopen for connection UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:40.108 DEBG [0] ExtentLiveRepair AckReady 1003, : downstairs
Sep 22 23:21:40.109 DEBG up_ds_listen was notified
Sep 22 23:21:40.109 DEBG up_ds_listen process 1003
Sep 22 23:21:40.109 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:40.109 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:40.109 DEBG Extent 0 id:1003 Done
Sep 22 23:21:40.109 DEBG Enqueue repair job 1004, : downstairs
Sep 22 23:21:40.109 INFO RE:0 Wait for result from NoOp command 1004:5
Sep 22 23:21:40.109 DEBG [0] 1004 Remove check skipped:{JobId(1001)} from deps:[JobId(1002), JobId(1003)], : downstairs
Sep 22 23:21:40.109 DEBG [0] 1004 Remove check < min repaired:1002 from deps:[JobId(1002), JobId(1003)], : downstairs
Sep 22 23:21:40.109 INFO [0] 1004 final dependency list [JobId(1002), JobId(1003)], : downstairs
Sep 22 23:21:40.109 DEBG Received NoOP 1004
Sep 22 23:21:40.109 DEBG Received NoOP 1004
Sep 22 23:21:40.109 DEBG Received NoOP 1004
Sep 22 23:21:40.109 DEBG Work of: LiveNoOp 1004
Sep 22 23:21:40.109 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:21:40.112 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:21:40.112 DEBG Work of: LiveNoOp 1004
Sep 22 23:21:40.112 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:21:40.114 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:21:40.114 DEBG Work of: LiveNoOp 1004
Sep 22 23:21:40.114 DEBG LiveNoOp  :1004 deps:[JobId(1002), JobId(1003)] res:true
Sep 22 23:21:40.118 DEBG LiveReopen:1005 extent 0 deps:[JobId(1002), JobId(1003), JobId(1004)] res:true
Sep 22 23:21:40.118 DEBG [0] ExtentLiveNoOp AckReady 1004, : downstairs
Sep 22 23:21:40.118 DEBG [0] ExtentLiveReopen AckReady 1005, : downstairs
Sep 22 23:21:40.118 DEBG up_ds_listen was notified
Sep 22 23:21:40.118 DEBG up_ds_listen process 1004
Sep 22 23:21:40.118 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:21:40.118 DEBG up_ds_listen process 1005
Sep 22 23:21:40.118 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:21:40.118 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:21:40.118 DEBG up_ds_listen was notified
Sep 22 23:21:40.118 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:21:40.118 DEBG Extent 0 id:1004 Done
Sep 22 23:21:40.118 INFO RE:0 Wait for result from reopen command 1005:6
Sep 22 23:21:40.118 DEBG Extent 0 id:1005 Done
Sep 22 23:21:40.119 INFO Start extent 1 repair, task: repair
Sep 22 23:21:40.119 DEBG RE:1 Repair extent begins
Sep 22 23:21:40.119 DEBG Create new job ids for 1, : downstairs
Sep 22 23:21:40.119 INFO RE:1 repair extent with ids 1006,1007,1008,1009 deps:[]
Sep 22 23:21:40.119 DEBG Enqueue repair job 1009, : downstairs
Sep 22 23:21:40.119 DEBG Enqueue repair job 1006, : downstairs
Sep 22 23:21:40.119 INFO RE:1 close id:1006 queued, notify DS
Sep 22 23:21:40.119 INFO RE:1 Wait for result from close command 1006:7
Sep 22 23:21:40.119 DEBG [0] 1006 Remove check skipped:{JobId(1001)} from deps:[], : downstairs
Sep 22 23:21:40.119 DEBG [0] 1006 Remove check < min repaired:1002 from deps:[], : downstairs
Sep 22 23:21:40.119 INFO [0] 1006 final dependency list [], : downstairs
Sep 22 23:21:40.119 DEBG [0] 1009 Remove check skipped:{JobId(1001)} from deps:[JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:21:40.119 DEBG [0] 1009 Remove check < min repaired:1002 from deps:[JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:21:40.119 INFO [0] 1009 final dependency list [JobId(1006), JobId(1007), JobId(1008)], : downstairs
Sep 22 23:21:40.119 DEBG Flush just extent 1 with f:3 and g:1
Sep 22 23:21:40.122 DEBG FlushClose:1006 extent 1 deps:[] res:true f:3 g:1
Sep 22 23:21:40.122 DEBG Flush just extent 1 with f:3 and g:1
Sep 22 23:21:40.123 DEBG FlushClose:1006 extent 1 deps:[] res:true f:3 g:1
Sep 22 23:21:40.126 DEBG JustClose :1006 extent 1 deps:[] res:true
Sep 22 23:21:40.126 DEBG [1] ELC got g:1 f:1 d:false
Sep 22 23:21:40.126 DEBG [2] ELC got g:1 f:1 d:false
Sep 22 23:21:40.126 DEBG [0] ELC got g:0 f:0 d:false
Sep 22 23:21:40.126 DEBG [0] ExtentFlushClose 1006 AckReady, : downstairs
Sep 22 23:21:40.126 DEBG up_ds_listen was notified
Sep 22 23:21:40.126 DEBG up_ds_listen process 1006
Sep 22 23:21:40.126 DEBG [A] ack job 1006:7, : downstairs
Sep 22 23:21:40.127 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:40.127 DEBG Extent 1 id:1006 Done
Sep 22 23:21:40.127 DEBG Get repair info for 2 source, : downstairs
Sep 22 23:21:40.127 DEBG Get repair info for 0 bad, : downstairs
Sep 22 23:21:40.127 INFO Repair for extent 1 s:2 d:[ClientId(0)], : downstairs
Sep 22 23:21:40.127 DEBG Enqueue repair job 1007, : downstairs
Sep 22 23:21:40.127 INFO RE:1 Wait for result from repair command 1007:8
Sep 22 23:21:40.127 DEBG [0] 1007 Remove check skipped:{JobId(1001)} from deps:[JobId(1006)], : downstairs
Sep 22 23:21:40.127 DEBG [0] 1007 Remove check < min repaired:1002 from deps:[JobId(1006)], : downstairs
Sep 22 23:21:40.127 INFO [0] 1007 final dependency list [JobId(1006)], : downstairs
Sep 22 23:21:40.127 DEBG Received NoOP 1007
Sep 22 23:21:40.127 DEBG Received NoOP 1007
Sep 22 23:21:40.127 DEBG Received ExtentLiveRepair 1007
Sep 22 23:21:40.127 DEBG Work of: LiveNoOp 1007
Sep 22 23:21:40.127 DEBG LiveNoOp  :1007 deps:[JobId(1006)] res:true
Sep 22 23:21:40.128 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:40.128 DEBG Work of: LiveNoOp 1007
Sep 22 23:21:40.128 DEBG LiveNoOp  :1007 deps:[JobId(1006)] res:true
Sep 22 23:21:40.128 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:40.128 DEBG ExtentLiveRepair: extent 1 sra:127.0.0.1:53407
Sep 22 23:21:40.128 INFO Created copy dir "/tmp/downstairs-kiLstbeI/00/000/001.copy"
Sep 22 23:21:40.214 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:40.214 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:40.214 INFO Opened existing region file "/tmp/downstairs-6BO8rCTH/region.json"
Sep 22 23:21:40.214 INFO Database read version 1
Sep 22 23:21:40.214 INFO Database write version 1
Sep 22 23:21:40.223 INFO accepted connection, remote_addr: 127.0.0.1:60945, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.223 TRCE incoming request, uri: /extent/1/files, method: GET, req_id: 822d821d-6e99-4e6b-919c-f3c7025fc2ae, remote_addr: 127.0.0.1:60945, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.224 INFO request completed, latency_us: 259, response_code: 200, uri: /extent/1/files, method: GET, req_id: 822d821d-6e99-4e6b-919c-f3c7025fc2ae, remote_addr: 127.0.0.1:60945, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.224 INFO eid:1 Found repair files: ["001", "001.db"]
Sep 22 23:21:40.224 TRCE incoming request, uri: /newextent/1/data, method: GET, req_id: 197e88fb-9c3b-4d88-917d-0477de97be12, remote_addr: 127.0.0.1:60945, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.225 INFO request completed, latency_us: 382, response_code: 200, uri: /newextent/1/data, method: GET, req_id: 197e88fb-9c3b-4d88-917d-0477de97be12, remote_addr: 127.0.0.1:60945, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.225 TRCE incoming request, uri: /newextent/1/db, method: GET, req_id: 1820bf47-7117-43da-b320-a55797afd8e6, remote_addr: 127.0.0.1:60945, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.226 INFO request completed, latency_us: 324, response_code: 200, uri: /newextent/1/db, method: GET, req_id: 1820bf47-7117-43da-b320-a55797afd8e6, remote_addr: 127.0.0.1:60945, local_addr: 127.0.0.1:53407, task: repair
Sep 22 23:21:40.227 INFO Repair files downloaded, move directory "/tmp/downstairs-kiLstbeI/00/000/001.copy" to "/tmp/downstairs-kiLstbeI/00/000/001.replace"
Sep 22 23:21:40.227 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000"
Sep 22 23:21:40.228 INFO Copy files from "/tmp/downstairs-kiLstbeI/00/000/001.replace" in "/tmp/downstairs-kiLstbeI/00/000"
Sep 22 23:21:40.228 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000/001"
Sep 22 23:21:40.228 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000/001.db"
Sep 22 23:21:40.228 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000"
Sep 22 23:21:40.228 INFO Move directory  "/tmp/downstairs-kiLstbeI/00/000/001.replace" to "/tmp/downstairs-kiLstbeI/00/000/001.completed"
Sep 22 23:21:40.228 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000"
Sep 22 23:21:40.228 DEBG fsync completed for: "/tmp/downstairs-kiLstbeI/00/000"
Sep 22 23:21:40.228 DEBG LiveRepair:1007 extent 1 deps:[JobId(1006)] res:true
Sep 22 23:21:40.228 WARN 1009 job ELiveReopen for connection UpstairsConnection { upstairs_id: 52fe8d40-333d-4b64-8663-2adf0476947f, session_id: fc0a5f14-ae91-4f48-aded-c2a3fecef8a0, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:21:40.228 DEBG [0] ExtentLiveRepair AckReady 1007, : downstairs
Sep 22 23:21:40.228 DEBG up_ds_listen was notified
Sep 22 23:21:40.228 DEBG up_ds_listen process 1007
Sep 22 23:21:40.228 DEBG [A] ack job 1007:8, : downstairs
Sep 22 23:21:40.228 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:40.228 DEBG Extent 1 id:1007 Done
Sep 22 23:21:40.229 DEBG Enqueue repair job 1008, : downstairs
Sep 22 23:21:40.229 INFO RE:1 Wait for result from NoOp command 1008:9
Sep 22 23:21:40.229 DEBG [0] 1008 Remove check skipped:{JobId(1001)} from deps:[JobId(1006), JobId(1007)], : downstairs
Sep 22 23:21:40.229 DEBG [0] 1008 Remove check < min repaired:1002 from deps:[JobId(1006), JobId(1007)], : downstairs
Sep 22 23:21:40.229 INFO [0] 1008 final dependency list [JobId(1006), JobId(1007)], : downstairs
Sep 22 23:21:40.229 DEBG Received NoOP 1008
Sep 22 23:21:40.229 DEBG Received NoOP 1008
Sep 22 23:21:40.229 DEBG Received NoOP 1008
Sep 22 23:21:40.229 DEBG Work of: LiveNoOp 1008
Sep 22 23:21:40.229 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:21:40.230 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:21:40.230 DEBG Work of: LiveNoOp 1008
Sep 22 23:21:40.230 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:21:40.232 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:21:40.232 DEBG Work of: LiveNoOp 1008
Sep 22 23:21:40.232 DEBG LiveNoOp  :1008 deps:[JobId(1006), JobId(1007)] res:true
Sep 22 23:21:40.234 DEBG LiveReopen:1009 extent 1 deps:[JobId(1006), JobId(1007), JobId(1008)] res:true
Sep 22 23:21:40.234 DEBG [0] ExtentLiveNoOp AckReady 1008, : downstairs
Sep 22 23:21:40.234 DEBG [0] ExtentLiveReopen AckReady 1009, : downstairs
Sep 22 23:21:40.234 DEBG up_ds_listen was notified
Sep 22 23:21:40.234 DEBG up_ds_listen process 1008
Sep 22 23:21:40.234 DEBG [A] ack job 1008:9, : downstairs
Sep 22 23:21:40.234 DEBG up_ds_listen process 1009
Sep 22 23:21:40.234 DEBG [A] ack job 1009:10, : downstairs
Sep 22 23:21:40.234 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:21:40.234 DEBG up_ds_listen was notified
Sep 22 23:21:40.234 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:21:40.234 DEBG Extent 1 id:1008 Done
Sep 22 23:21:40.234 INFO RE:1 Wait for result from reopen command 1009:10
Sep 22 23:21:40.234 DEBG Extent 1 id:1009 Done
Sep 22 23:21:40.234 DEBG IO Flush 1010 has deps [JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)]
Sep 22 23:21:40.234 INFO LiveRepair final flush submitted
Sep 22 23:21:40.234 DEBG [0] 1010 Remove check skipped:{JobId(1001)} from deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:21:40.234 DEBG [0] 1010 Remove check < min repaired:1002 from deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:21:40.234 INFO [0] 1010 final dependency list [JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)], : downstairs
Sep 22 23:21:40.235 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:21:40.235 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:21:40.235 DEBG Flush     :1010 extent_limit Some(1) deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002)] res:true f:4 g:1
Sep 22 23:21:40.235 DEBG up_ds_listen was notified
Sep 22 23:21:40.235 DEBG up_ds_listen process 1010
Sep 22 23:21:40.235 DEBG [A] ack job 1010:11, : downstairs
Sep 22 23:21:40.235 DEBG [rc] retire 1010 clears [JobId(1002), JobId(1003), JobId(1004), JobId(1005), JobId(1006), JobId(1007), JobId(1008), JobId(1009), JobId(1010)], : downstairs
Sep 22 23:21:40.235 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:40.235 INFO LiveRepair final flush completed
Sep 22 23:21:40.235 INFO [0] 52fe8d40-333d-4b64-8663-2adf0476947f (fc0a5f14-ae91-4f48-aded-c2a3fecef8a0) LiveRepair Active Active ds_transition to Active
Sep 22 23:21:40.235 INFO [0] Transition from LiveRepair to Active
Sep 22 23:21:40.235 WARN Live Repair returns Ok(())
Sep 22 23:21:40.240 INFO UUID: 3184f538-c5cb-4487-be43-e6cb9becc716
Sep 22 23:21:40.240 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:40.240 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:40.240 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:40.240 INFO Using address: 127.0.0.1:46698, task: main
Sep 22 23:21:40.241 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:40.241 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:38546, task: repair
Sep 22 23:21:40.241 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:38546, task: repair
Sep 22 23:21:40.241 INFO listening, local_addr: 127.0.0.1:38546, task: repair
Sep 22 23:21:40.241 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:38546, task: repair
Sep 22 23:21:40.241 INFO Using repair address: 127.0.0.1:38546, task: main
Sep 22 23:21:40.241 INFO No SSL acceptor configured, task: main
Sep 22 23:21:40.249 INFO UUID: b63118d3-af4f-496f-b02c-9f9b92d23429
Sep 22 23:21:40.249 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:40.249 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:40.249 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:40.249 INFO Using address: 127.0.0.1:60546, task: main
Sep 22 23:21:40.249 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:40.249 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:63823, task: repair
Sep 22 23:21:40.249 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:63823, task: repair
Sep 22 23:21:40.249 INFO listening, local_addr: 127.0.0.1:63823, task: repair
Sep 22 23:21:40.250 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:63823, task: repair
Sep 22 23:21:40.250 INFO Using repair address: 127.0.0.1:63823, task: main
Sep 22 23:21:40.250 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.296313343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:46176","component":"dropshot"}
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.296386768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:47828","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:46176","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.296518022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"listen IP: 127.0.0.1:47828","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.29656651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:40.297 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:40.297 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:40.297 INFO Opened existing region file "/tmp/downstairs-AuFNRJc7/region.json"
Sep 22 23:21:40.297 INFO Database read version 1
Sep 22 23:21:40.297 INFO Database write version 1
Sep 22 23:21:40.343 INFO UUID: e1ba5200-2113-47b5-8eab-dc553c509bb6
Sep 22 23:21:40.343 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:40.343 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:40.343 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:40.343 INFO Using address: 127.0.0.1:53040, task: main
Sep 22 23:21:40.344 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:40.344 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:65165, task: repair
Sep 22 23:21:40.344 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:65165, task: repair
Sep 22 23:21:40.344 INFO listening, local_addr: 127.0.0.1:65165, task: repair
Sep 22 23:21:40.344 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:65165, task: repair
Sep 22 23:21:40.344 INFO Using repair address: 127.0.0.1:65165, task: main
Sep 22 23:21:40.344 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.364167379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:47396","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:47396","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.364384077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:40.385 INFO UUID: 121dfa8b-dae0-4803-ad33-1b8e5b36d846
Sep 22 23:21:40.385 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:40.385 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:40.385 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:40.385 INFO Using address: 127.0.0.1:60454, task: main
Sep 22 23:21:40.385 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:40.386 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35500, task: repair
Sep 22 23:21:40.386 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35500, task: repair
Sep 22 23:21:40.386 INFO listening, local_addr: 127.0.0.1:35500, task: repair
Sep 22 23:21:40.386 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35500, task: repair
Sep 22 23:21:40.386 INFO Using repair address: 127.0.0.1:35500, task: main
Sep 22 23:21:40.386 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.399894577Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:52393","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:52393","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.400109004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:40.499 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.500224622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:47828","component":"dropshot","remote_addr":"127.0.0.1:35008"}
{"msg":"no entry exists for volume 07fcaddf-9197-4aa4-a6a7-ae8fc7c212ec, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.501069057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.501344564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.501415655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.501451271Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 6d97a177-f79d-4abd-9fa5-a1779824805b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.501486886Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible 6d97a177-f79d-4abd-9fa5-a1779824805b has session id: b26982c0-3685-4f78-910a-02e75227794c","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.501522816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:33519","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.501652325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:52265","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.5017988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:46698","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.501939388Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502065339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502115256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502148206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 07fcaddf-9197-4aa4-a6a7-ae8fc7c212ec constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502357107Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"6d97a177-f79d-4abd-9fa5-a1779824805b active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502628475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 6d97a177-f79d-4abd-9fa5-a1779824805b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502733621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:33519 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502783604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 6d97a177-f79d-4abd-9fa5-a1779824805b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502824191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:52265 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502865325Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 6d97a177-f79d-4abd-9fa5-a1779824805b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502901463Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:46698 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.502937568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.502 INFO accepted connection from 127.0.0.1:50318, task: main
Sep 22 23:21:40.503 INFO accepted connection from 127.0.0.1:34759, task: main
Sep 22 23:21:40.503 INFO accepted connection from 127.0.0.1:56062, task: main
Sep 22 23:21:40.503 INFO Connection request from 6d97a177-f79d-4abd-9fa5-a1779824805b with version 4, task: proc
Sep 22 23:21:40.503 INFO upstairs UpstairsConnection { upstairs_id: 6d97a177-f79d-4abd-9fa5-a1779824805b, session_id: ae25a180-0753-4d68-9404-5e9bb9308f43, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:40.503 INFO Connection request from 6d97a177-f79d-4abd-9fa5-a1779824805b with version 4, task: proc
Sep 22 23:21:40.503 INFO upstairs UpstairsConnection { upstairs_id: 6d97a177-f79d-4abd-9fa5-a1779824805b, session_id: ae25a180-0753-4d68-9404-5e9bb9308f43, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:40.503 INFO Connection request from 6d97a177-f79d-4abd-9fa5-a1779824805b with version 4, task: proc
Sep 22 23:21:40.503 INFO upstairs UpstairsConnection { upstairs_id: 6d97a177-f79d-4abd-9fa5-a1779824805b, session_id: ae25a180-0753-4d68-9404-5e9bb9308f43, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 6d97a177-f79d-4abd-9fa5-a1779824805b (ae25a180-0753-4d68-9404-5e9bb9308f43) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.50376731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.503811898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session ae25a180-0753-4d68-9404-5e9bb9308f43","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.503847207Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 6d97a177-f79d-4abd-9fa5-a1779824805b (ae25a180-0753-4d68-9404-5e9bb9308f43) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.503882528Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.503914754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session ae25a180-0753-4d68-9404-5e9bb9308f43","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.503948683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 6d97a177-f79d-4abd-9fa5-a1779824805b (ae25a180-0753-4d68-9404-5e9bb9308f43) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.503991179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.504024684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session ae25a180-0753-4d68-9404-5e9bb9308f43","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.504058492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.504 INFO UpstairsConnection { upstairs_id: 6d97a177-f79d-4abd-9fa5-a1779824805b, session_id: ae25a180-0753-4d68-9404-5e9bb9308f43, gen: 1 } is now active (read-write)
Sep 22 23:21:40.504 INFO UpstairsConnection { upstairs_id: 6d97a177-f79d-4abd-9fa5-a1779824805b, session_id: ae25a180-0753-4d68-9404-5e9bb9308f43, gen: 1 } is now active (read-write)
Sep 22 23:21:40.504 INFO UpstairsConnection { upstairs_id: 6d97a177-f79d-4abd-9fa5-a1779824805b, session_id: ae25a180-0753-4d68-9404-5e9bb9308f43, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:33519 has UUID 6e2bf597-efa8-46d7-84b3-855cd0183cc5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.505442345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 6e2bf597-efa8-46d7-84b3-855cd0183cc5, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.505497744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"6d97a177-f79d-4abd-9fa5-a1779824805b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.505533202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:52265 has UUID 13d4e6fc-362c-415f-bc7d-909deb752536","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.505568385Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 13d4e6fc-362c-415f-bc7d-909deb752536, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.5056084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"6d97a177-f79d-4abd-9fa5-a1779824805b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.505643327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:46698 has UUID 3184f538-c5cb-4487-be43-e6cb9becc716","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.505683841Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 3184f538-c5cb-4487-be43-e6cb9becc716, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.505732877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"6d97a177-f79d-4abd-9fa5-a1779824805b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.505781541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.515 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.515858348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:46176","component":"dropshot","remote_addr":"127.0.0.1:35475"}
{"msg":"no entry exists for volume ed764ae9-e3ce-441f-8278-67469b5636e4, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.516545106Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.516782701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.51683174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.516856649Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 585127d2-6fce-4638-823a-9e437ec79e5b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.516880561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible 585127d2-6fce-4638-823a-9e437ec79e5b has session id: fe8dcfff-e2c4-48d5-bda2-4cb07880027f","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.516904933Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:37108","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.517049353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:35718","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.517199412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:60546","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.517339706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.517501454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.517545516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.517578006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume ed764ae9-e3ce-441f-8278-67469b5636e4 constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.517808805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.518109809Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.518 INFO accepted connection from 127.0.0.1:43703, task: main
{"msg":"[0] 585127d2-6fce-4638-823a-9e437ec79e5b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.518329189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:37108 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.518379267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 585127d2-6fce-4638-823a-9e437ec79e5b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.518415797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:35718 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.518453285Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 585127d2-6fce-4638-823a-9e437ec79e5b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.51848706Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:60546 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.518521983Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.518 INFO accepted connection from 127.0.0.1:60507, task: main
Sep 22 23:21:40.518 INFO accepted connection from 127.0.0.1:49103, task: main
Sep 22 23:21:40.518 INFO Connection request from 585127d2-6fce-4638-823a-9e437ec79e5b with version 4, task: proc
Sep 22 23:21:40.518 INFO upstairs UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:40.518 INFO Connection request from 585127d2-6fce-4638-823a-9e437ec79e5b with version 4, task: proc
Sep 22 23:21:40.519 INFO upstairs UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:40.519 INFO Connection request from 585127d2-6fce-4638-823a-9e437ec79e5b with version 4, task: proc
Sep 22 23:21:40.519 INFO upstairs UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.519358243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.519405622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 3a227d09-1b16-48c0-8c37-0e4189fb3e80","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.519441413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.519478327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.519511757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 3a227d09-1b16-48c0-8c37-0e4189fb3e80","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.519553531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.519591532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.519624372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 3a227d09-1b16-48c0-8c37-0e4189fb3e80","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.519658743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.520 INFO UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } is now active (read-write)
Sep 22 23:21:40.520 INFO UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } is now active (read-write)
Sep 22 23:21:40.520 INFO UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:37108 has UUID aa05d4f9-364f-4612-a579-613b38785c79","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.521228371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: aa05d4f9-364f-4612-a579-613b38785c79, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.521306091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.521344542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:35718 has UUID 9c7d4f0a-db95-4538-bb7e-7b1ed1936555","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.521379986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 9c7d4f0a-db95-4538-bb7e-7b1ed1936555, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.52142075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.521455693Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:60546 has UUID b63118d3-af4f-496f-b02c-9f9b92d23429","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.521490681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: b63118d3-af4f-496f-b02c-9f9b92d23429, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.521531442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.521572176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.521 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.524 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.526 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.539 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.542 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.545 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.550 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.551 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.551 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 6d97a177-f79d-4abd-9fa5-a1779824805b (ae25a180-0753-4d68-9404-5e9bb9308f43) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.55243668Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.552504425Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.552539924Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.552573379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 6d97a177-f79d-4abd-9fa5-a1779824805b (ae25a180-0753-4d68-9404-5e9bb9308f43) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.552622444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.552657438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.552690481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.552722613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 6d97a177-f79d-4abd-9fa5-a1779824805b (ae25a180-0753-4d68-9404-5e9bb9308f43) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.552866762Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.552908801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.552942296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.552974757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:33519 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553008116Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"6d97a177-f79d-4abd-9fa5-a1779824805b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553051481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553086467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553120759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553156181Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553190017Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553223936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553282191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553322409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553358742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553398113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.5534393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553480096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553517514Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553555915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553592959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553628557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.55366404Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"6d97a177-f79d-4abd-9fa5-a1779824805b is now active with session: ae25a180-0753-4d68-9404-5e9bb9308f43","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553706779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"6d97a177-f79d-4abd-9fa5-a1779824805b Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553742621Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553777233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.55381026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:52265 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553844787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"6d97a177-f79d-4abd-9fa5-a1779824805b Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.55388026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553912391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:46698 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553946412Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"6d97a177-f79d-4abd-9fa5-a1779824805b Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.553980205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.554012592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.554045279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.554077587Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.554110329Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.55414275Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.554175042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.554206444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.554238601Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.55429546Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.554335109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 07fcaddf-9197-4aa4-a6a7-ae8fc7c212ec activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.554369481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 07fcaddf-9197-4aa4-a6a7-ae8fc7c212ec constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.554403539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.55443542Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/07fcaddf-9197-4aa4-a6a7-ae8fc7c212ec","method":"POST","req_id":"0744ae93-fe5e-48b4-8805-16081d978cf1","remote_addr":"127.0.0.1:35008","local_addr":"127.0.0.1:47828","component":"dropshot","latency_us":52941,"response_code":"200"}
Sep 22 23:21:40.579 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.580 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.582 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.58321504Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.583285189Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.583326806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.583369772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.58362087Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.583675358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.583724593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.583769897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584021465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584073062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.584113692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584153204Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:37108 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584200953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584242509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584287274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.58433222Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584380611Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584423842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584474414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584521864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584563148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584602772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584643952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584683139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584727149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584769861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584815132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584860769Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584904536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584950102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b is now active with session: 3a227d09-1b16-48c0-8c37-0e4189fb3e80","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.584992394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585031949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585071107Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585109581Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:35718 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585148855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585188489Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585226823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:60546 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585265339Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585304186Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585341488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585379648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585418411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585453637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585487597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585523999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585559493Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585595637Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585637725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585674405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume ed764ae9-e3ce-441f-8278-67469b5636e4 activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.58571178Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume ed764ae9-e3ce-441f-8278-67469b5636e4 constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585749486Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.585787314Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/ed764ae9-e3ce-441f-8278-67469b5636e4","method":"POST","req_id":"c8f070d8-292e-4ce5-a796-8dc6973edfe1","remote_addr":"127.0.0.1:35475","local_addr":"127.0.0.1:46176","component":"dropshot","latency_us":68837,"response_code":"200"}
Sep 22 23:21:40.587 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.58831077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:47396","component":"dropshot","remote_addr":"127.0.0.1:39199"}
{"msg":"no entry exists for volume 1eb9cf2b-fac5-48fd-86a9-e21629057f3e, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.588842502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.589061415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.589212548Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.589317086Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 40e06af7-e69f-43e2-974b-bb00dc5e8960","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.5893823Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible 40e06af7-e69f-43e2-974b-bb00dc5e8960 has session id: fa831e57-f170-4210-8b5e-b43db9390e2c","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.589478731Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:55664","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.589597496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:37034","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.589716228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
The guest has requested activation
{"msg":"[2] connecting to 127.0.0.1:53040","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.589824056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.589938408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
Sep 22 23:21:40.590 INFO accepted connection from 127.0.0.1:49946, task: main
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.590070125Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.590 INFO accepted connection from 127.0.0.1:45897, task: main
Sep 22 23:21:40.590 INFO accepted connection from 127.0.0.1:36645, task: main
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.590219434Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 1eb9cf2b-fac5-48fd-86a9-e21629057f3e constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.590309605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.590 INFO Connection request from 40e06af7-e69f-43e2-974b-bb00dc5e8960 with version 4, task: proc
Sep 22 23:21:40.590 INFO upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } connected, version 4, task: proc
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.590373411Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.590 INFO Connection request from 40e06af7-e69f-43e2-974b-bb00dc5e8960 with version 4, task: proc
Sep 22 23:21:40.590 INFO upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.59048962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
Sep 22 23:21:40.590 INFO Connection request from 40e06af7-e69f-43e2-974b-bb00dc5e8960 with version 4, task: proc
Sep 22 23:21:40.590 INFO upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } connected, version 4, task: proc
{"msg":"[0] Proc runs for 127.0.0.1:55664 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.590582228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.590673184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:37034 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.590800074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.590904387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:53040 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.59103184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.591137052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.591274985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.591 INFO UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } is now active (read-write)
{"msg":"[0] client is_active_req TRUE, promote! session d90ddc5f-b441-432b-bb73-5ab29b73ba5b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.59139485Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.591 INFO UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } is now active (read-write)
{"msg":"[1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.591470751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.591514936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.591 INFO UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } is now active (read-write)
{"msg":"[1] client is_active_req TRUE, promote! session d90ddc5f-b441-432b-bb73-5ab29b73ba5b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.591558133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.591601789Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.59164077Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session d90ddc5f-b441-432b-bb73-5ab29b73ba5b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.59168137Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] downstairs client at 127.0.0.1:55664 has UUID b740b481-b810-4686-a6ad-5e7a129e9669","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.591929835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: b740b481-b810-4686-a6ad-5e7a129e9669, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.591984456Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.592025902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:37034 has UUID efa5c554-210a-4196-815d-fa3d1dfbb858","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.592066906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: efa5c554-210a-4196-815d-fa3d1dfbb858, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.592114041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.592160371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:53040 has UUID e1ba5200-2113-47b5-8eab-dc553c509bb6","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.592202198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: e1ba5200-2113-47b5-8eab-dc553c509bb6, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.592248895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.592288846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.603 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.605 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.607 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.619 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.619650515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:52393","component":"dropshot","remote_addr":"127.0.0.1:57584"}
{"msg":"no entry exists for volume 3251378f-ea13-40c9-9d37-7f38d07fbf35, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.620425273Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.620726834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.620806232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.620845034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: cf566b9e-aff0-4665-9031-98e179d159a4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.620882544Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible cf566b9e-aff0-4665-9031-98e179d159a4 has session id: f610c67f-9d19-426f-9069-eec75dc91553","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.620920655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:40057","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.620983093Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:65451","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.621172451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:60454","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.621308612Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.621396966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.621445523Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.621484014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 3251378f-ea13-40c9-9d37-7f38d07fbf35 constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.621696371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"cf566b9e-aff0-4665-9031-98e179d159a4 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.621984889Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] cf566b9e-aff0-4665-9031-98e179d159a4 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.622135847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:40057 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.622185464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] cf566b9e-aff0-4665-9031-98e179d159a4 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.622222816Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:65451 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.622260474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] cf566b9e-aff0-4665-9031-98e179d159a4 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.6222966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:60454 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.622334148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.622 INFO accepted connection from 127.0.0.1:42340, task: main
Sep 22 23:21:40.622 INFO accepted connection from 127.0.0.1:42583, task: main
Sep 22 23:21:40.622 INFO accepted connection from 127.0.0.1:40516, task: main
Sep 22 23:21:40.622 INFO Connection request from cf566b9e-aff0-4665-9031-98e179d159a4 with version 4, task: proc
Sep 22 23:21:40.622 INFO upstairs UpstairsConnection { upstairs_id: cf566b9e-aff0-4665-9031-98e179d159a4, session_id: e7612fa3-27f9-4d09-a4f3-b6224148140c, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:40.622 INFO Connection request from cf566b9e-aff0-4665-9031-98e179d159a4 with version 4, task: proc
Sep 22 23:21:40.622 INFO upstairs UpstairsConnection { upstairs_id: cf566b9e-aff0-4665-9031-98e179d159a4, session_id: e7612fa3-27f9-4d09-a4f3-b6224148140c, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:40.623 INFO Connection request from cf566b9e-aff0-4665-9031-98e179d159a4 with version 4, task: proc
Sep 22 23:21:40.623 INFO upstairs UpstairsConnection { upstairs_id: cf566b9e-aff0-4665-9031-98e179d159a4, session_id: e7612fa3-27f9-4d09-a4f3-b6224148140c, gen: 1 } connected, version 4, task: proc
{"msg":"[0] cf566b9e-aff0-4665-9031-98e179d159a4 (e7612fa3-27f9-4d09-a4f3-b6224148140c) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.623258508Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.623306525Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session e7612fa3-27f9-4d09-a4f3-b6224148140c","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.623344977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] cf566b9e-aff0-4665-9031-98e179d159a4 (e7612fa3-27f9-4d09-a4f3-b6224148140c) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.623383793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.623418749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session e7612fa3-27f9-4d09-a4f3-b6224148140c","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.623455265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] cf566b9e-aff0-4665-9031-98e179d159a4 (e7612fa3-27f9-4d09-a4f3-b6224148140c) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.623498292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.623536343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session e7612fa3-27f9-4d09-a4f3-b6224148140c","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.623573815Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.624 INFO UpstairsConnection { upstairs_id: cf566b9e-aff0-4665-9031-98e179d159a4, session_id: e7612fa3-27f9-4d09-a4f3-b6224148140c, gen: 1 } is now active (read-write)
Sep 22 23:21:40.624 INFO UpstairsConnection { upstairs_id: cf566b9e-aff0-4665-9031-98e179d159a4, session_id: e7612fa3-27f9-4d09-a4f3-b6224148140c, gen: 1 } is now active (read-write)
Sep 22 23:21:40.624 INFO UpstairsConnection { upstairs_id: cf566b9e-aff0-4665-9031-98e179d159a4, session_id: e7612fa3-27f9-4d09-a4f3-b6224148140c, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:40057 has UUID 1f0dc9eb-57e9-42b1-a5c8-0ed1810eb444","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.625106547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 1f0dc9eb-57e9-42b1-a5c8-0ed1810eb444, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.625162977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"cf566b9e-aff0-4665-9031-98e179d159a4 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.625201048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:65451 has UUID dfb16c90-27d7-4a45-bd56-019dbea3b0bb","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.625238969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: dfb16c90-27d7-4a45-bd56-019dbea3b0bb, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.625295288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"cf566b9e-aff0-4665-9031-98e179d159a4 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.625335241Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:60454 has UUID 121dfa8b-dae0-4803-ad33-1b8e5b36d846","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.625372688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 121dfa8b-dae0-4803-ad33-1b8e5b36d846, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.625416217Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"cf566b9e-aff0-4665-9031-98e179d159a4 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.62545265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.625 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.626 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.627 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.62793123Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.627980155Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.628018592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628057425Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.62814678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.62819005Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.628225684Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628259576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628424265Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628476676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.628513219Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.62855326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:55664 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628590937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.62862781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628664028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628699692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628737238Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628773495Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628811266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.62885302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628889959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628925629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628963425Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.628997366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.62903196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629072475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629112834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.62914866Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629183428Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629218349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 is now active with session: d90ddc5f-b441-432b-bb73-5ab29b73ba5b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629256357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629292592Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629328031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629365061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:37034 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629403965Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629440427Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629474664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:53040 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629510089Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629545751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629579552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629613624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629652047Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629688043Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629722474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629762943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629797747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629832108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629866188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629900497Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 1eb9cf2b-fac5-48fd-86a9-e21629057f3e activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629940707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 1eb9cf2b-fac5-48fd-86a9-e21629057f3e constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.629978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.630011909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/1eb9cf2b-fac5-48fd-86a9-e21629057f3e","method":"POST","req_id":"19e8e4b7-584d-4860-9bb9-d77596a10c8e","remote_addr":"127.0.0.1:39199","local_addr":"127.0.0.1:47396","component":"dropshot","latency_us":40515,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.630078806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/1eb9cf2b-fac5-48fd-86a9-e21629057f3e/import_from_url","method":"POST","req_id":"40b20567-b3b8-4142-9c18-058bb40bfd73","remote_addr":"127.0.0.1:39199","local_addr":"127.0.0.1:47396","component":"dropshot","latency_us":226,"response_code":"200"}
Sep 22 23:21:40.643 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.645 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.648 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:40.678 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.680 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:40.681 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] cf566b9e-aff0-4665-9031-98e179d159a4 (e7612fa3-27f9-4d09-a4f3-b6224148140c) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.682444881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.682507106Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.68254344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.682578446Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] cf566b9e-aff0-4665-9031-98e179d159a4 (e7612fa3-27f9-4d09-a4f3-b6224148140c) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.68284661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.682903249Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.682940342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.682974913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] cf566b9e-aff0-4665-9031-98e179d159a4 (e7612fa3-27f9-4d09-a4f3-b6224148140c) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683257955Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683307883Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:40.683345507Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683380092Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:40057 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683415952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"cf566b9e-aff0-4665-9031-98e179d159a4 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683452794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683488906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683525074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683565056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683605532Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683642449Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683680308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683716149Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683752262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683790436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.68382553Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683865414Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683900451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683934359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.683969003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684003698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.68403843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"cf566b9e-aff0-4665-9031-98e179d159a4 is now active with session: e7612fa3-27f9-4d09-a4f3-b6224148140c","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684076518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"cf566b9e-aff0-4665-9031-98e179d159a4 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684115004Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.68415452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684189749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:65451 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684225266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"cf566b9e-aff0-4665-9031-98e179d159a4 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.68426114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.68429541Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:60454 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684330674Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"cf566b9e-aff0-4665-9031-98e179d159a4 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684366289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684412661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684450337Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684485377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684519695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684554066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.6845885Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684622979Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684659089Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684698742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684733958Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 3251378f-ea13-40c9-9d37-7f38d07fbf35 activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684769709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 3251378f-ea13-40c9-9d37-7f38d07fbf35 constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.68480599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.684839843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/3251378f-ea13-40c9-9d37-7f38d07fbf35","method":"POST","req_id":"cc4a97f6-c0bc-4193-a279-a2f075f7f767","remote_addr":"127.0.0.1:57584","local_addr":"127.0.0.1:52393","component":"dropshot","latency_us":64198,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.68559101Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/3251378f-ea13-40c9-9d37-7f38d07fbf35/import_from_url","method":"POST","req_id":"21ae3786-e768-471e-904a-7755a2d1296d","remote_addr":"127.0.0.1:57584","local_addr":"127.0.0.1:52393","component":"dropshot","latency_us":394,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.700200686Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/e14704dd-f376-471e-a250-3ebae85bb495/is_finished","method":"GET","req_id":"b91e04c4-0783-4764-86b6-ba3828e8be17","remote_addr":"127.0.0.1:39199","local_addr":"127.0.0.1:47396","component":"dropshot","latency_us":228,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.762118812Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/7165c87e-0953-466c-822f-e26bef4d9d7d/is_finished","method":"GET","req_id":"1d1fe7f5-7d19-4477-a4bd-246857fb6569","remote_addr":"127.0.0.1:57584","local_addr":"127.0.0.1:52393","component":"dropshot","latency_us":234,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.807094819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/07fcaddf-9197-4aa4-a6a7-ae8fc7c212ec/bulk_write","method":"POST","req_id":"d2b8adcd-68bf-4c7d-a905-f102a2a96cac","remote_addr":"127.0.0.1:35008","local_addr":"127.0.0.1:47828","component":"dropshot","latency_us":223917,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.909879419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/ed764ae9-e3ce-441f-8278-67469b5636e4/bulk_write","method":"POST","req_id":"1f6a8a20-1e59-4998-ad8a-00c90bab92aa","remote_addr":"127.0.0.1:35475","local_addr":"127.0.0.1:46176","component":"dropshot","latency_us":278122,"response_code":"204"}
{"msg":"detach removing entry for volume ed764ae9-e3ce-441f-8278-67469b5636e4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.913698538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume ed764ae9-e3ce-441f-8278-67469b5636e4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:40.913755052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:40.916 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:40.920 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:40.945 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:40.949 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:40.976 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:40.980 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:41.005 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:41.021 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:41.037 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:41.042 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:41.042 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:41.042 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.043908846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.043981074Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.044143502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/ed764ae9-e3ce-441f-8278-67469b5636e4","method":"DELETE","req_id":"2de4f42e-7986-4816-831b-385fccf593e4","remote_addr":"127.0.0.1:35475","local_addr":"127.0.0.1:46176","component":"dropshot","latency_us":130716,"response_code":"204"}
Sep 22 23:21:41.044 INFO Upstairs starts
Sep 22 23:21:41.044 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:41.044 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:41.044 INFO Crucible stats registered with UUID: 585127d2-6fce-4638-823a-9e437ec79e5b
Sep 22 23:21:41.044 INFO Crucible 585127d2-6fce-4638-823a-9e437ec79e5b has session id: 1ae42173-6784-460f-b521-f55c07c51ee3
Sep 22 23:21:41.044 INFO [0] connecting to 127.0.0.1:37108, looper: 0
Sep 22 23:21:41.044 INFO [1] connecting to 127.0.0.1:35718, looper: 1
Sep 22 23:21:41.045 INFO [2] connecting to 127.0.0.1:60546, looper: 2
Sep 22 23:21:41.045 INFO up_listen starts, task: up_listen
Sep 22 23:21:41.045 INFO Wait for all three downstairs to come online
Sep 22 23:21:41.045 INFO Flush timeout: 0.5
Sep 22 23:21:41.045 INFO accepted connection from 127.0.0.1:58406, task: main
Sep 22 23:21:41.045 INFO accepted connection from 127.0.0.1:59580, task: main
Sep 22 23:21:41.045 INFO [0] 585127d2-6fce-4638-823a-9e437ec79e5b looper connected, looper: 0
Sep 22 23:21:41.045 INFO [0] Proc runs for 127.0.0.1:37108 in state New
Sep 22 23:21:41.045 INFO [1] 585127d2-6fce-4638-823a-9e437ec79e5b looper connected, looper: 1
Sep 22 23:21:41.045 INFO [1] Proc runs for 127.0.0.1:35718 in state New
Sep 22 23:21:41.045 INFO [2] 585127d2-6fce-4638-823a-9e437ec79e5b looper connected, looper: 2
Sep 22 23:21:41.045 INFO [2] Proc runs for 127.0.0.1:60546 in state New
Sep 22 23:21:41.045 INFO accepted connection from 127.0.0.1:43230, task: main
Sep 22 23:21:41.045 INFO Connection request from 585127d2-6fce-4638-823a-9e437ec79e5b with version 4, task: proc
Sep 22 23:21:41.045 INFO upstairs UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:41.046 INFO Connection request from 585127d2-6fce-4638-823a-9e437ec79e5b with version 4, task: proc
Sep 22 23:21:41.046 INFO upstairs UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:41.046 INFO Connection request from 585127d2-6fce-4638-823a-9e437ec79e5b with version 4, task: proc
Sep 22 23:21:41.046 INFO upstairs UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:41.046 INFO [0] 585127d2-6fce-4638-823a-9e437ec79e5b (32007434-41a4-434b-a63f-6381f23a145d) New New New ds_transition to WaitActive
Sep 22 23:21:41.046 INFO [0] Transition from New to WaitActive
Sep 22 23:21:41.046 INFO [1] 585127d2-6fce-4638-823a-9e437ec79e5b (32007434-41a4-434b-a63f-6381f23a145d) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:41.046 INFO [1] Transition from New to WaitActive
Sep 22 23:21:41.046 INFO [2] 585127d2-6fce-4638-823a-9e437ec79e5b (32007434-41a4-434b-a63f-6381f23a145d) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:41.046 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:21:41.046 INFO 585127d2-6fce-4638-823a-9e437ec79e5b active request set
Sep 22 23:21:41.046 INFO [0] received activate with gen 2
Sep 22 23:21:41.046 INFO [0] client got ds_active_rx, promote! session 32007434-41a4-434b-a63f-6381f23a145d
Sep 22 23:21:41.046 INFO [1] received activate with gen 2
Sep 22 23:21:41.046 INFO [1] client got ds_active_rx, promote! session 32007434-41a4-434b-a63f-6381f23a145d
Sep 22 23:21:41.046 INFO [2] received activate with gen 2
Sep 22 23:21:41.046 INFO [2] client got ds_active_rx, promote! session 32007434-41a4-434b-a63f-6381f23a145d
Sep 22 23:21:41.046 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } to UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 }
Sep 22 23:21:41.046 WARN Signaling to UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } thread that UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } is being promoted (read-write)
Sep 22 23:21:41.047 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } to UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 }
Sep 22 23:21:41.047 WARN Signaling to UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } thread that UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } is being promoted (read-write)
Sep 22 23:21:41.047 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } to UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 }
Sep 22 23:21:41.047 WARN Signaling to UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 } thread that UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } is being promoted (read-write)
Sep 22 23:21:41.047 WARN Another upstairs UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 }, task: main
Sep 22 23:21:41.047 INFO UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } is now active (read-write)
Sep 22 23:21:41.047 WARN Another upstairs UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 }, task: main
Sep 22 23:21:41.047 INFO UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } is now active (read-write)
Sep 22 23:21:41.047 WARN Another upstairs UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 3a227d09-1b16-48c0-8c37-0e4189fb3e80, gen: 1 }, task: main
Sep 22 23:21:41.048 INFO UpstairsConnection { upstairs_id: 585127d2-6fce-4638-823a-9e437ec79e5b, session_id: 32007434-41a4-434b-a63f-6381f23a145d, gen: 2 } is now active (read-write)
Sep 22 23:21:41.048 INFO connection (127.0.0.1:43703): all done
Sep 22 23:21:41.048 INFO connection (127.0.0.1:60507): all done
Sep 22 23:21:41.048 INFO connection (127.0.0.1:49103): all done
{"msg":"[0] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) cmd_loop saw YouAreNoLongerActive 585127d2-6fce-4638-823a-9e437ec79e5b 32007434-41a4-434b-a63f-6381f23a145d 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:41.048510913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) Active Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.048566022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.048605188Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b set inactive, session 3a227d09-1b16-48c0-8c37-0e4189fb3e80","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.048645442Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:37108: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:41.048693795Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] 585127d2-6fce-4638-823a-9e437ec79e5b Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.048737641Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 585127d2-6fce-4638-823a-9e437ec79e5b connection to 127.0.0.1:37108 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.048776968Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) cmd_loop saw YouAreNoLongerActive 585127d2-6fce-4638-823a-9e437ec79e5b 32007434-41a4-434b-a63f-6381f23a145d 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:41.048823075Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) Disconnected Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.048864831Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.048901936Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b set inactive, session 3a227d09-1b16-48c0-8c37-0e4189fb3e80","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.048940925Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:35718: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:41.048980842Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] 585127d2-6fce-4638-823a-9e437ec79e5b Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049022622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 585127d2-6fce-4638-823a-9e437ec79e5b connection to 127.0.0.1:35718 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049061196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) cmd_loop saw YouAreNoLongerActive 585127d2-6fce-4638-823a-9e437ec79e5b 32007434-41a4-434b-a63f-6381f23a145d 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:41.049106746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 585127d2-6fce-4638-823a-9e437ec79e5b (3a227d09-1b16-48c0-8c37-0e4189fb3e80) Disconnected Disconnected Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049148228Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049185247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b set inactive, session 3a227d09-1b16-48c0-8c37-0e4189fb3e80","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049224279Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:60546: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:41.049269467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] 585127d2-6fce-4638-823a-9e437ec79e5b Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049312832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 585127d2-6fce-4638-823a-9e437ec79e5b connection to 127.0.0.1:60546 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049351349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:41.049389765Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:37108 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049426835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.04946509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:37108 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049502384Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:41.049 INFO [0] downstairs client at 127.0.0.1:37108 has UUID aa05d4f9-364f-4612-a579-613b38785c79
Sep 22 23:21:41.049 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: aa05d4f9-364f-4612-a579-613b38785c79, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[1] 127.0.0.1:35718 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.04953972Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:41.049 INFO 585127d2-6fce-4638-823a-9e437ec79e5b WaitActive WaitActive WaitActive
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049578518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:35718 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049615586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:41.049 INFO [1] downstairs client at 127.0.0.1:35718 has UUID 9c7d4f0a-db95-4538-bb7e-7b1ed1936555
Sep 22 23:21:41.049 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 9c7d4f0a-db95-4538-bb7e-7b1ed1936555, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[2] 127.0.0.1:60546 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049652323Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:41.049 INFO 585127d2-6fce-4638-823a-9e437ec79e5b WaitActive WaitActive WaitActive
{"msg":"585127d2-6fce-4638-823a-9e437ec79e5b Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.04969076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:60546 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.049727978Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:41.049 INFO [2] downstairs client at 127.0.0.1:60546 has UUID b63118d3-af4f-496f-b02c-9f9b92d23429
Sep 22 23:21:41.049 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: b63118d3-af4f-496f-b02c-9f9b92d23429, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[1] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:41.049769659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:41.049 INFO 585127d2-6fce-4638-823a-9e437ec79e5b WaitActive WaitActive WaitActive
{"msg":"[2] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:41.049807374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:41.057 INFO Current flush_numbers [0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:41.058 INFO Current flush_numbers [0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:41.059 INFO Current flush_numbers [0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:41.072 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:41.073 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:41.074 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:41.074 INFO [0] 585127d2-6fce-4638-823a-9e437ec79e5b (32007434-41a4-434b-a63f-6381f23a145d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:41.074 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:41.074 WARN [0] new RM replaced this: None
Sep 22 23:21:41.074 INFO [0] Starts reconcile loop
Sep 22 23:21:41.074 INFO [1] 585127d2-6fce-4638-823a-9e437ec79e5b (32007434-41a4-434b-a63f-6381f23a145d) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:41.075 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:41.075 WARN [1] new RM replaced this: None
Sep 22 23:21:41.075 INFO [1] Starts reconcile loop
Sep 22 23:21:41.075 INFO [2] 585127d2-6fce-4638-823a-9e437ec79e5b (32007434-41a4-434b-a63f-6381f23a145d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:41.075 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:41.075 WARN [2] new RM replaced this: None
Sep 22 23:21:41.075 INFO [2] Starts reconcile loop
Sep 22 23:21:41.075 INFO [0] 127.0.0.1:37108 task reports connection:true
Sep 22 23:21:41.075 INFO 585127d2-6fce-4638-823a-9e437ec79e5b WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:41.075 INFO [0]R flush_numbers[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:41.075 INFO [0]R generation[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:41.075 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:41.075 INFO [1]R flush_numbers[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:41.075 INFO [1]R generation[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:41.075 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:41.075 INFO [2]R flush_numbers[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:41.075 INFO [2]R generation[0..12]: [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:41.075 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:41.075 INFO Max found gen is 2
Sep 22 23:21:41.075 INFO Generation requested: 2 >= found:2
Sep 22 23:21:41.075 INFO Next flush: 2
Sep 22 23:21:41.075 INFO All extents match
Sep 22 23:21:41.075 INFO No downstairs repair required
Sep 22 23:21:41.075 INFO No initial repair work was required
Sep 22 23:21:41.075 INFO Set Downstairs and Upstairs active
Sep 22 23:21:41.075 INFO 585127d2-6fce-4638-823a-9e437ec79e5b is now active with session: 32007434-41a4-434b-a63f-6381f23a145d
Sep 22 23:21:41.075 INFO 585127d2-6fce-4638-823a-9e437ec79e5b Set Active after no repair
Sep 22 23:21:41.075 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:41.075 INFO Set check for repair
Sep 22 23:21:41.075 INFO [1] 127.0.0.1:35718 task reports connection:true
Sep 22 23:21:41.075 INFO 585127d2-6fce-4638-823a-9e437ec79e5b Active Active Active
Sep 22 23:21:41.075 INFO Set check for repair
Sep 22 23:21:41.075 INFO [2] 127.0.0.1:60546 task reports connection:true
Sep 22 23:21:41.075 INFO 585127d2-6fce-4638-823a-9e437ec79e5b Active Active Active
Sep 22 23:21:41.075 INFO Set check for repair
Sep 22 23:21:41.075 INFO [0] received reconcile message
Sep 22 23:21:41.075 INFO [0] All repairs completed, exit
Sep 22 23:21:41.075 INFO [0] Starts cmd_loop
Sep 22 23:21:41.075 INFO [1] received reconcile message
Sep 22 23:21:41.075 INFO [1] All repairs completed, exit
Sep 22 23:21:41.075 INFO [1] Starts cmd_loop
Sep 22 23:21:41.076 INFO [2] received reconcile message
Sep 22 23:21:41.076 INFO [2] All repairs completed, exit
Sep 22 23:21:41.076 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:41.077 DEBG IO Read  1000 has deps []
Sep 22 23:21:41.087 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:41.101 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:41.120 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:41.426 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:41.591 DEBG [1] Read already AckReady 1000, : downstairs
{"msg":"Checking if live repair is needed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.68466855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.684756117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.702736661Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.702803565Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.703830758Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/e14704dd-f376-471e-a250-3ebae85bb495/is_finished","method":"GET","req_id":"748e1c09-92e3-4fba-b7d0-a9eaa2c05c5d","remote_addr":"127.0.0.1:39199","local_addr":"127.0.0.1:47396","component":"dropshot","latency_us":385,"response_code":"200"}
Sep 22 23:21:41.725 WARN f712941d-0577-43e4-8a2e-7814ec270c09 request to replace downstairs 127.0.0.1:46295 with 127.0.0.1:47135
Sep 22 23:21:41.725 INFO f712941d-0577-43e4-8a2e-7814ec270c09 found new target: 127.0.0.1:47135 at 0
Sep 22 23:21:41.725 INFO Downstairs replacement completed
Sep 22 23:21:41.725 DEBG IO Read  1011 has deps []
Sep 22 23:21:41.726 DEBG Read      :1011 deps:[] res:true
Sep 22 23:21:41.727 DEBG Read      :1011 deps:[] res:true
Sep 22 23:21:41.727 DEBG Read      :1011 deps:[] res:true
Sep 22 23:21:41.732 DEBG [1] Read AckReady 1011, : downstairs
Sep 22 23:21:41.735 DEBG [2] Read already AckReady 1011, : downstairs
Sep 22 23:21:41.738 DEBG [0] Read already AckReady 1011, : downstairs
Sep 22 23:21:41.738 DEBG up_ds_listen was notified
Sep 22 23:21:41.738 DEBG up_ds_listen process 1011
Sep 22 23:21:41.738 DEBG [A] ack job 1011:12, : downstairs
Sep 22 23:21:41.739 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::integration_test_guest_replace_downstairs ... ok
Sep 22 23:21:41.756 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:41.757 DEBG up_ds_listen was notified
Sep 22 23:21:41.757 DEBG up_ds_listen process 1000
Sep 22 23:21:41.757 DEBG [A] ack job 1000:1, : downstairs
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.76432589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/7165c87e-0953-466c-822f-e26bef4d9d7d/is_finished","method":"GET","req_id":"45f054bd-5ad6-433f-a9e6-0701595805cf","remote_addr":"127.0.0.1:57584","local_addr":"127.0.0.1:52393","component":"dropshot","latency_us":315,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.767371166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/07fcaddf-9197-4aa4-a6a7-ae8fc7c212ec/bulk_read","method":"POST","req_id":"5dc0da9e-7211-4ba5-afda-0f61e6fc3dad","remote_addr":"127.0.0.1:35008","local_addr":"127.0.0.1:47828","component":"dropshot","latency_us":956875,"response_code":"200"}
{"msg":"Checking if live repair is needed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.767651658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.767697042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:41.776 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:41.777 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:41.777 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:41.785 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"detach removing entry for volume 07fcaddf-9197-4aa4-a6a7-ae8fc7c212ec","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.802605221Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume 07fcaddf-9197-4aa4-a6a7-ae8fc7c212ec","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.802673254Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:41.803 DEBG Flush     :1003 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:21:41.803 DEBG Flush     :1003 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:21:41.803 DEBG Flush     :1003 extent_limit None deps:[] res:true f:2 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.804202711Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"6d97a177-f79d-4abd-9fa5-a1779824805b set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.804254225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.804506371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/07fcaddf-9197-4aa4-a6a7-ae8fc7c212ec","method":"DELETE","req_id":"580bc9aa-16aa-4105-ac00-971b48886428","remote_addr":"127.0.0.1:35008","local_addr":"127.0.0.1:47828","component":"dropshot","latency_us":2098,"response_code":"204"}
Sep 22 23:21:41.811 INFO Checking if live repair is needed
Sep 22 23:21:41.811 INFO No Live Repair required at this time
Sep 22 23:21:41.844 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:41.869 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:41.894 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:41.907 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:41.907 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:41.907 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
test test::test_pantry_bulk_write_max_chunk_size ... ok
Sep 22 23:21:41.944 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.944 INFO Created new region file "/tmp/downstairs-we6eLCZU/region.json"
Sep 22 23:21:41.948 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.948 INFO Opened existing region file "/tmp/downstairs-we6eLCZU/region.json"
Sep 22 23:21:41.948 INFO Database read version 1
Sep 22 23:21:41.948 INFO Database write version 1
Sep 22 23:21:41.949 INFO UUID: 5bd06377-7334-447c-a337-f5f05f8d6c8d
Sep 22 23:21:41.949 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:41.949 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:41.949 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:41.949 INFO Using address: 127.0.0.1:56105, task: main
Sep 22 23:21:41.950 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:41.950 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53964, task: repair
Sep 22 23:21:41.950 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53964, task: repair
Sep 22 23:21:41.950 INFO listening, local_addr: 127.0.0.1:53964, task: repair
Sep 22 23:21:41.950 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53964, task: repair
Sep 22 23:21:41.950 INFO Using repair address: 127.0.0.1:53964, task: main
Sep 22 23:21:41.950 INFO No SSL acceptor configured, task: main
Sep 22 23:21:41.950 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.950 INFO Created new region file "/tmp/downstairs-2o9xo501/region.json"
Sep 22 23:21:41.954 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.954 INFO Opened existing region file "/tmp/downstairs-2o9xo501/region.json"
Sep 22 23:21:41.954 INFO Database read version 1
Sep 22 23:21:41.954 INFO Database write version 1
Sep 22 23:21:41.955 INFO UUID: 9bb0c711-bf0b-4c97-9224-93a1bd568273
Sep 22 23:21:41.955 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:41.956 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:41.956 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:41.956 INFO Using address: 127.0.0.1:34504, task: main
Sep 22 23:21:41.956 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:41.956 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:51030, task: repair
Sep 22 23:21:41.956 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:51030, task: repair
Sep 22 23:21:41.956 INFO listening, local_addr: 127.0.0.1:51030, task: repair
Sep 22 23:21:41.956 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:51030, task: repair
Sep 22 23:21:41.956 INFO Using repair address: 127.0.0.1:51030, task: main
Sep 22 23:21:41.956 INFO No SSL acceptor configured, task: main
Sep 22 23:21:41.956 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.957 INFO Created new region file "/tmp/downstairs-QafMFymO/region.json"
Sep 22 23:21:41.960 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.960 INFO Opened existing region file "/tmp/downstairs-QafMFymO/region.json"
Sep 22 23:21:41.960 INFO Database read version 1
Sep 22 23:21:41.960 INFO Database write version 1
Sep 22 23:21:41.961 INFO UUID: 0a385deb-9c37-4b6c-9838-b74475ae3131
Sep 22 23:21:41.961 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:41.961 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:41.961 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:41.961 INFO Using address: 127.0.0.1:33240, task: main
Sep 22 23:21:41.962 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:41.962 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62996, task: repair
Sep 22 23:21:41.962 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62996, task: repair
Sep 22 23:21:41.962 INFO listening, local_addr: 127.0.0.1:62996, task: repair
Sep 22 23:21:41.962 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62996, task: repair
Sep 22 23:21:41.962 INFO Using repair address: 127.0.0.1:62996, task: main
Sep 22 23:21:41.962 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.96407371Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:58561","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:58561","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.964183705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
test test::test_pantry_bulk_read_max_chunk_size ... ok
Sep 22 23:21:41.978 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.979 INFO Created new region file "/tmp/downstairs-m4GfdQ6u/region.json"
Sep 22 23:21:41.981 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.981 INFO Opened existing region file "/tmp/downstairs-m4GfdQ6u/region.json"
Sep 22 23:21:41.981 INFO Database read version 1
Sep 22 23:21:41.981 INFO Database write version 1
Sep 22 23:21:41.981 INFO UUID: 1dc6ff1a-e3e5-49bb-96af-2f8e2384da86
Sep 22 23:21:41.981 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:41.981 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:41.981 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:41.981 INFO Using address: 127.0.0.1:40394, task: main
Sep 22 23:21:41.982 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:41.982 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:42941, task: repair
Sep 22 23:21:41.982 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:42941, task: repair
Sep 22 23:21:41.982 INFO listening, local_addr: 127.0.0.1:42941, task: repair
Sep 22 23:21:41.982 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:42941, task: repair
Sep 22 23:21:41.982 INFO Using repair address: 127.0.0.1:42941, task: main
Sep 22 23:21:41.982 INFO No SSL acceptor configured, task: main
Sep 22 23:21:41.982 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.982 INFO Created new region file "/tmp/downstairs-snkVG7mj/region.json"
Sep 22 23:21:41.984 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.984 INFO Opened existing region file "/tmp/downstairs-snkVG7mj/region.json"
Sep 22 23:21:41.984 INFO Database read version 1
Sep 22 23:21:41.984 INFO Database write version 1
Sep 22 23:21:41.985 INFO UUID: 6b825ea1-626e-494d-9a24-d66b39d14dbc
Sep 22 23:21:41.985 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:41.985 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:41.985 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:41.985 INFO Using address: 127.0.0.1:44812, task: main
Sep 22 23:21:41.985 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:41.985 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:47303, task: repair
Sep 22 23:21:41.985 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:47303, task: repair
Sep 22 23:21:41.985 INFO listening, local_addr: 127.0.0.1:47303, task: repair
Sep 22 23:21:41.985 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:47303, task: repair
Sep 22 23:21:41.985 INFO Using repair address: 127.0.0.1:47303, task: main
Sep 22 23:21:41.985 INFO No SSL acceptor configured, task: main
Sep 22 23:21:41.985 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.986 INFO Created new region file "/tmp/downstairs-WgIExdEe/region.json"
Sep 22 23:21:41.988 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:41.988 INFO Opened existing region file "/tmp/downstairs-WgIExdEe/region.json"
Sep 22 23:21:41.988 INFO Database read version 1
Sep 22 23:21:41.988 INFO Database write version 1
Sep 22 23:21:41.988 INFO UUID: 19cc35ab-e1e8-4b0e-9882-b11c9bb545fa
Sep 22 23:21:41.988 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:41.988 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:41.988 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:41.988 INFO Using address: 127.0.0.1:50704, task: main
Sep 22 23:21:41.989 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:41.989 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62111, task: repair
Sep 22 23:21:41.989 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62111, task: repair
Sep 22 23:21:41.989 INFO listening, local_addr: 127.0.0.1:62111, task: repair
Sep 22 23:21:41.989 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62111, task: repair
Sep 22 23:21:41.989 INFO Using repair address: 127.0.0.1:62111, task: main
Sep 22 23:21:41.989 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.990558712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:63226","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:41.990726379Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:41.991 WARN 52fe8d40-333d-4b64-8663-2adf0476947f request to replace downstairs 127.0.0.1:38958 with 127.0.0.1:62901
Sep 22 23:21:41.991 INFO 52fe8d40-333d-4b64-8663-2adf0476947f found new target: 127.0.0.1:62901 at 0
Downstairs replacement completed
Sep 22 23:21:41.991 DEBG IO Read  1011 has deps []
Sep 22 23:21:41.992 DEBG Read      :1011 deps:[] res:true
Sep 22 23:21:41.993 DEBG Read      :1011 deps:[] res:true
Sep 22 23:21:41.993 DEBG Read      :1011 deps:[] res:true
Sep 22 23:21:41.998 DEBG [1] Read AckReady 1011, : downstairs
Sep 22 23:21:42.001 DEBG [2] Read already AckReady 1011, : downstairs
Sep 22 23:21:42.004 DEBG [0] Read already AckReady 1011, : downstairs
Sep 22 23:21:42.004 DEBG up_ds_listen was notified
Sep 22 23:21:42.004 DEBG up_ds_listen process 1011
Sep 22 23:21:42.004 DEBG [A] ack job 1011:12, : downstairs
Sep 22 23:21:42.004 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::integration_test_volume_replace_downstairs ... ok
Sep 22 23:21:42.011 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.011 INFO Created new region file "/tmp/downstairs-9LeQywgd/region.json"
Sep 22 23:21:42.014 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.014 INFO Opened existing region file "/tmp/downstairs-9LeQywgd/region.json"
Sep 22 23:21:42.014 INFO Database read version 1
Sep 22 23:21:42.014 INFO Database write version 1
Sep 22 23:21:42.015 INFO UUID: 981e485a-3d34-445d-9adb-f9e13ff6f329
Sep 22 23:21:42.015 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.015 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.015 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.015 INFO Using address: 127.0.0.1:42682, task: main
Sep 22 23:21:42.016 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.016 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:60320, task: repair
Sep 22 23:21:42.016 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:60320, task: repair
Sep 22 23:21:42.016 INFO listening, local_addr: 127.0.0.1:60320, task: repair
Sep 22 23:21:42.016 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:60320, task: repair
Sep 22 23:21:42.016 INFO Using repair address: 127.0.0.1:60320, task: main
Sep 22 23:21:42.016 INFO No SSL acceptor configured, task: main
Sep 22 23:21:42.016 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.016 INFO Created new region file "/tmp/downstairs-KFBQqDry/region.json"
Sep 22 23:21:42.019 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.019 INFO Opened existing region file "/tmp/downstairs-KFBQqDry/region.json"
Sep 22 23:21:42.019 INFO Database read version 1
Sep 22 23:21:42.019 INFO Database write version 1
Sep 22 23:21:42.020 INFO UUID: c77d45f9-2dd1-4698-958d-b4ab9ce4ffee
Sep 22 23:21:42.020 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.020 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.020 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.020 INFO Using address: 127.0.0.1:61724, task: main
Sep 22 23:21:42.020 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.020 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:54833, task: repair
Sep 22 23:21:42.021 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:54833, task: repair
Sep 22 23:21:42.021 INFO listening, local_addr: 127.0.0.1:54833, task: repair
Sep 22 23:21:42.021 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:54833, task: repair
Sep 22 23:21:42.021 INFO Using repair address: 127.0.0.1:54833, task: main
Sep 22 23:21:42.021 INFO No SSL acceptor configured, task: main
Sep 22 23:21:42.021 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.021 INFO Created new region file "/tmp/downstairs-4kPeDRfm/region.json"
Sep 22 23:21:42.024 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.024 INFO Opened existing region file "/tmp/downstairs-4kPeDRfm/region.json"
Sep 22 23:21:42.024 INFO Database read version 1
Sep 22 23:21:42.024 INFO Database write version 1
Sep 22 23:21:42.025 INFO UUID: 300935ac-634a-4338-b182-cfe66d8d8948
Sep 22 23:21:42.025 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.025 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.025 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.025 INFO Using address: 127.0.0.1:47055, task: main
Sep 22 23:21:42.025 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.025 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:62348, task: repair
Sep 22 23:21:42.025 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:62348, task: repair
Sep 22 23:21:42.025 INFO listening, local_addr: 127.0.0.1:62348, task: repair
Sep 22 23:21:42.025 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:62348, task: repair
Sep 22 23:21:42.026 INFO Using repair address: 127.0.0.1:62348, task: main
Sep 22 23:21:42.026 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.027886467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:33810","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:33810","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.028081802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:42.124 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.124 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.125 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.125477635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:58561","component":"dropshot","remote_addr":"127.0.0.1:41754"}
{"msg":"no entry exists for volume 6d13f2e2-dfe6-43d7-aeab-f413816a137e, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.126324713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.126614749Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.126694975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.126733854Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.126772269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 has session id: cf6d7f60-3bbd-4eef-a7da-e75edf81adf5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.12681767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:56105","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.126869895Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:34504","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.127031159Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:33240","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.12716316Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.127226929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.127268487Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.127303392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 6d13f2e2-dfe6-43d7-aeab-f413816a137e constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.127509777Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.127811205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.127 INFO accepted connection from 127.0.0.1:39184, task: main
Sep 22 23:21:42.127 INFO accepted connection from 127.0.0.1:46433, task: main
Sep 22 23:21:42.128 INFO accepted connection from 127.0.0.1:62619, task: main
{"msg":"[0] 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.128191085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:56105 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.128242516Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.128280278Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:34504 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.128318335Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.128355599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:33240 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.128398688Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.128 INFO Connection request from 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 with version 4, task: proc
Sep 22 23:21:42.128 INFO upstairs UpstairsConnection { upstairs_id: 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0, session_id: 2e08e8ff-b29e-4152-8a8e-7239daeea0f1, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.128 INFO Connection request from 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 with version 4, task: proc
Sep 22 23:21:42.128 INFO upstairs UpstairsConnection { upstairs_id: 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0, session_id: 2e08e8ff-b29e-4152-8a8e-7239daeea0f1, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.128 INFO Connection request from 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 with version 4, task: proc
Sep 22 23:21:42.128 INFO upstairs UpstairsConnection { upstairs_id: 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0, session_id: 2e08e8ff-b29e-4152-8a8e-7239daeea0f1, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 (2e08e8ff-b29e-4152-8a8e-7239daeea0f1) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.128991798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.129047431Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 2e08e8ff-b29e-4152-8a8e-7239daeea0f1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.129087305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 (2e08e8ff-b29e-4152-8a8e-7239daeea0f1) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.129127356Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.129168443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 2e08e8ff-b29e-4152-8a8e-7239daeea0f1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.129207342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 (2e08e8ff-b29e-4152-8a8e-7239daeea0f1) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.129247303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.129282754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.129 INFO UpstairsConnection { upstairs_id: 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0, session_id: 2e08e8ff-b29e-4152-8a8e-7239daeea0f1, gen: 1 } is now active (read-write)
{"msg":"[2] client is_active_req TRUE, promote! session 2e08e8ff-b29e-4152-8a8e-7239daeea0f1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.129320229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.129 INFO UpstairsConnection { upstairs_id: 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0, session_id: 2e08e8ff-b29e-4152-8a8e-7239daeea0f1, gen: 1 } is now active (read-write)
Sep 22 23:21:42.129 INFO UpstairsConnection { upstairs_id: 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0, session_id: 2e08e8ff-b29e-4152-8a8e-7239daeea0f1, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:56105 has UUID 5bd06377-7334-447c-a337-f5f05f8d6c8d","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.12999298Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 5bd06377-7334-447c-a337-f5f05f8d6c8d, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.130050333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.13008877Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:34504 has UUID 9bb0c711-bf0b-4c97-9224-93a1bd568273","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.130130199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9bb0c711-bf0b-4c97-9224-93a1bd568273, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.130178148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.130223345Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:33240 has UUID 0a385deb-9c37-4b6c-9838-b74475ae3131","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.130262415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 0a385deb-9c37-4b6c-9838-b74475ae3131, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.130305814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.130342462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.130 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.130 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.130 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.131 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.131 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.131 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 (2e08e8ff-b29e-4152-8a8e-7239daeea0f1) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.131645759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.13169494Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.131731406Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.13177002Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 (2e08e8ff-b29e-4152-8a8e-7239daeea0f1) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.1318132Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.131850367Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.13188966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.131925827Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 (2e08e8ff-b29e-4152-8a8e-7239daeea0f1) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.131965832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132001986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.132047598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132087811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:56105 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132124306Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132161251Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132195888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.1322307Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132265246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132300755Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132340635Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132376701Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132411369Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132446214Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132480556Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132514687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132549578Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132585205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132623803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132659356Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132694194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.13273599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 is now active with session: 2e08e8ff-b29e-4152-8a8e-7239daeea0f1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132775166Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132811519Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132847415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132887187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:34504 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.13292447Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132960524Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.132995084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:33240 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.13303015Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133065888Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133099902Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133136393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133175875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133211227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.13324602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133280358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.13331547Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133350409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133385247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133428246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 6d13f2e2-dfe6-43d7-aeab-f413816a137e activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133468945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 6d13f2e2-dfe6-43d7-aeab-f413816a137e constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133506458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133540622Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/6d13f2e2-dfe6-43d7-aeab-f413816a137e","method":"POST","req_id":"9b3e1787-4434-4959-bf7a-a3ae6b2ae36f","remote_addr":"127.0.0.1:41754","local_addr":"127.0.0.1:58561","component":"dropshot","latency_us":6580,"response_code":"200"}
{"msg":"flush with snap requested","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.133609365Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.134 ERRO Snapshot request received on unsupported binary
Sep 22 23:21:42.134 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
Sep 22 23:21:42.134 ERRO Snapshot request received on unsupported binary
Sep 22 23:21:42.134 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
Sep 22 23:21:42.134 ERRO Snapshot request received on unsupported binary
Sep 22 23:21:42.134 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.134985103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/6d13f2e2-dfe6-43d7-aeab-f413816a137e/snapshot","method":"POST","req_id":"8f6d35ef-46dd-4149-9cfc-0e8bc774721b","remote_addr":"127.0.0.1:41754","local_addr":"127.0.0.1:58561","component":"dropshot","latency_us":1713,"response_code":"204"}
{"msg":"detach removing entry for volume 6d13f2e2-dfe6-43d7-aeab-f413816a137e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.135675134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume 6d13f2e2-dfe6-43d7-aeab-f413816a137e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.135728361Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.136 DEBG Flush     :1001 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:21:42.136 DEBG Flush     :1001 extent_limit None deps:[] res:true f:2 g:1
Sep 22 23:21:42.136 DEBG Flush     :1001 extent_limit None deps:[] res:true f:2 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.136992392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"2b87c9f1-3f50-4bc0-9661-27a6adb02fb0 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.137041692Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.137259562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/6d13f2e2-dfe6-43d7-aeab-f413816a137e","method":"DELETE","req_id":"06fbc838-40cc-498e-996e-1858bd3f1bb1","remote_addr":"127.0.0.1:41754","local_addr":"127.0.0.1:58561","component":"dropshot","latency_us":1686,"response_code":"204"}
test test::test_pantry_snapshot ... ok
Sep 22 23:21:42.144 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.144 INFO Created new region file "/tmp/downstairs-WW0WMgjz/region.json"
Sep 22 23:21:42.147 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.147 INFO Opened existing region file "/tmp/downstairs-WW0WMgjz/region.json"
Sep 22 23:21:42.147 INFO Database read version 1
Sep 22 23:21:42.147 INFO Database write version 1
Sep 22 23:21:42.148 INFO UUID: 101c5e08-1317-4b60-a031-62766e2cdb1f
Sep 22 23:21:42.148 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.148 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.148 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.148 INFO Using address: 127.0.0.1:49499, task: main
Sep 22 23:21:42.149 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.149 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:40788, task: repair
Sep 22 23:21:42.149 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:40788, task: repair
Sep 22 23:21:42.149 INFO listening, local_addr: 127.0.0.1:40788, task: repair
Sep 22 23:21:42.149 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:40788, task: repair
Sep 22 23:21:42.149 INFO Using repair address: 127.0.0.1:40788, task: main
Sep 22 23:21:42.149 INFO No SSL acceptor configured, task: main
Sep 22 23:21:42.149 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.149 INFO Created new region file "/tmp/downstairs-QUbX2kxB/region.json"
Sep 22 23:21:42.152 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.152 INFO Opened existing region file "/tmp/downstairs-QUbX2kxB/region.json"
Sep 22 23:21:42.152 INFO Database read version 1
Sep 22 23:21:42.152 INFO Database write version 1
Sep 22 23:21:42.153 INFO UUID: 44286653-4082-4ed7-a812-3555da359c78
Sep 22 23:21:42.153 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.153 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.153 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.153 INFO Using address: 127.0.0.1:56926, task: main
Sep 22 23:21:42.153 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.153 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:39647, task: repair
Sep 22 23:21:42.154 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:39647, task: repair
Sep 22 23:21:42.154 INFO listening, local_addr: 127.0.0.1:39647, task: repair
Sep 22 23:21:42.154 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:39647, task: repair
Sep 22 23:21:42.154 INFO Using repair address: 127.0.0.1:39647, task: main
Sep 22 23:21:42.154 INFO No SSL acceptor configured, task: main
Sep 22 23:21:42.154 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.154 INFO Created new region file "/tmp/downstairs-iu0Qdi78/region.json"
Sep 22 23:21:42.157 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.157 INFO Opened existing region file "/tmp/downstairs-iu0Qdi78/region.json"
Sep 22 23:21:42.157 INFO Database read version 1
Sep 22 23:21:42.157 INFO Database write version 1
Sep 22 23:21:42.158 INFO UUID: d4d95f2d-aa01-4319-8829-47ddcf12c9ca
Sep 22 23:21:42.158 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.158 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.158 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.158 INFO Using address: 127.0.0.1:46734, task: main
Sep 22 23:21:42.158 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.158 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58207, task: repair
Sep 22 23:21:42.158 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58207, task: repair
Sep 22 23:21:42.158 INFO listening, local_addr: 127.0.0.1:58207, task: repair
Sep 22 23:21:42.159 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58207, task: repair
Sep 22 23:21:42.159 INFO Using repair address: 127.0.0.1:58207, task: main
Sep 22 23:21:42.159 INFO No SSL acceptor configured, task: main
note: configured to log to "/dev/stdout"
Sep 22 23:21:42.160 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.160 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.160 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.160625041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot","remote_addr":"127.0.0.1:35139"}
{"msg":"no entry exists for volume fc92b52b-99b5-4d79-ab62-675ee18fb434, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161117048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161254595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161330313Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161411634Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"listen IP: 127.0.0.1:59371","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161438407Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161453492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 94d7a72a-673e-42de-882f-9dbeb845d1f4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161492451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible 94d7a72a-673e-42de-882f-9dbeb845d1f4 has session id: 9d22304a-a5f9-4f90-8c1f-fec9500ab362","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161531517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:40394","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161566472Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:44812","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161629349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:50704","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161785672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.16183184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161870223Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.161909212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume fc92b52b-99b5-4d79-ab62-675ee18fb434 constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.162138977Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"94d7a72a-673e-42de-882f-9dbeb845d1f4 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.162455576Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.162 INFO accepted connection from 127.0.0.1:60751, task: main
Sep 22 23:21:42.162 INFO accepted connection from 127.0.0.1:55531, task: main
Sep 22 23:21:42.162 INFO accepted connection from 127.0.0.1:40108, task: main
{"msg":"[0] 94d7a72a-673e-42de-882f-9dbeb845d1f4 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.162841374Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:40394 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.162892557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 94d7a72a-673e-42de-882f-9dbeb845d1f4 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.16292963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:44812 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.162966714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 94d7a72a-673e-42de-882f-9dbeb845d1f4 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.163001952Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:50704 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.163037198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.163 INFO Connection request from 94d7a72a-673e-42de-882f-9dbeb845d1f4 with version 4, task: proc
Sep 22 23:21:42.163 INFO upstairs UpstairsConnection { upstairs_id: 94d7a72a-673e-42de-882f-9dbeb845d1f4, session_id: 34608033-2e41-41aa-a912-4128bf340849, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.163 INFO Connection request from 94d7a72a-673e-42de-882f-9dbeb845d1f4 with version 4, task: proc
Sep 22 23:21:42.163 INFO upstairs UpstairsConnection { upstairs_id: 94d7a72a-673e-42de-882f-9dbeb845d1f4, session_id: 34608033-2e41-41aa-a912-4128bf340849, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.163 INFO Connection request from 94d7a72a-673e-42de-882f-9dbeb845d1f4 with version 4, task: proc
Sep 22 23:21:42.163 INFO upstairs UpstairsConnection { upstairs_id: 94d7a72a-673e-42de-882f-9dbeb845d1f4, session_id: 34608033-2e41-41aa-a912-4128bf340849, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 94d7a72a-673e-42de-882f-9dbeb845d1f4 (34608033-2e41-41aa-a912-4128bf340849) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.163608957Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.163659943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 34608033-2e41-41aa-a912-4128bf340849","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.163702623Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 94d7a72a-673e-42de-882f-9dbeb845d1f4 (34608033-2e41-41aa-a912-4128bf340849) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.163742779Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.163777973Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 34608033-2e41-41aa-a912-4128bf340849","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.16381448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 94d7a72a-673e-42de-882f-9dbeb845d1f4 (34608033-2e41-41aa-a912-4128bf340849) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.163853381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.163 INFO UpstairsConnection { upstairs_id: 94d7a72a-673e-42de-882f-9dbeb845d1f4, session_id: 34608033-2e41-41aa-a912-4128bf340849, gen: 1 } is now active (read-write)
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.163889308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 34608033-2e41-41aa-a912-4128bf340849","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.163925687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.163 INFO UpstairsConnection { upstairs_id: 94d7a72a-673e-42de-882f-9dbeb845d1f4, session_id: 34608033-2e41-41aa-a912-4128bf340849, gen: 1 } is now active (read-write)
Sep 22 23:21:42.164 INFO UpstairsConnection { upstairs_id: 94d7a72a-673e-42de-882f-9dbeb845d1f4, session_id: 34608033-2e41-41aa-a912-4128bf340849, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:40394 has UUID 1dc6ff1a-e3e5-49bb-96af-2f8e2384da86","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.164615319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 1dc6ff1a-e3e5-49bb-96af-2f8e2384da86, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.164673269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"94d7a72a-673e-42de-882f-9dbeb845d1f4 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.164711282Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:44812 has UUID 6b825ea1-626e-494d-9a24-d66b39d14dbc","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.164748624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 6b825ea1-626e-494d-9a24-d66b39d14dbc, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.164798529Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"94d7a72a-673e-42de-882f-9dbeb845d1f4 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.164835482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:50704 has UUID 19cc35ab-e1e8-4b0e-9882-b11c9bb545fa","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.164870835Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 19cc35ab-e1e8-4b0e-9882-b11c9bb545fa, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.164911741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"94d7a72a-673e-42de-882f-9dbeb845d1f4 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.164946054Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.164 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.165 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.165 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.165 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.165 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.165 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 94d7a72a-673e-42de-882f-9dbeb845d1f4 (34608033-2e41-41aa-a912-4128bf340849) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166263435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166314595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.166355781Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166396617Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 94d7a72a-673e-42de-882f-9dbeb845d1f4 (34608033-2e41-41aa-a912-4128bf340849) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166442257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166482399Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.166522242Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166559607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 94d7a72a-673e-42de-882f-9dbeb845d1f4 (34608033-2e41-41aa-a912-4128bf340849) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166600246Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166641174Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.166676389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166711794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:40394 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166748095Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"94d7a72a-673e-42de-882f-9dbeb845d1f4 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.1667851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166820048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166854539Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166887754Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166920797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.166953256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.1669859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167018194Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.1670509Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167083045Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167115418Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167147703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167181629Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167221226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167259113Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167306505Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167348281Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"94d7a72a-673e-42de-882f-9dbeb845d1f4 is now active with session: 34608033-2e41-41aa-a912-4128bf340849","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167389165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"94d7a72a-673e-42de-882f-9dbeb845d1f4 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167430649Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167468774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167503536Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:44812 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167537719Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"94d7a72a-673e-42de-882f-9dbeb845d1f4 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167572153Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167605405Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:50704 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167639775Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"94d7a72a-673e-42de-882f-9dbeb845d1f4 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167676598Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167711675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167747013Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167782041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167816124Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167850804Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167883764Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.167916803Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.16795499Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.16798891Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.168021274Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume fc92b52b-99b5-4d79-ab62-675ee18fb434 activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.168055041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume fc92b52b-99b5-4d79-ab62-675ee18fb434 constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.168094943Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.168134297Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434","method":"POST","req_id":"189f22d2-19b5-44f2-993a-6c0d67bfb933","remote_addr":"127.0.0.1:35139","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":6199,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.172410851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot","remote_addr":"127.0.0.1:60149"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.172473821Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot","remote_addr":"127.0.0.1:62526"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.172524252Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot","remote_addr":"127.0.0.1:63083"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.172573158Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot","remote_addr":"127.0.0.1:46607"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.172618136Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot","remote_addr":"127.0.0.1:34605"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.1726599Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot","remote_addr":"127.0.0.1:64643"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.172700712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot","remote_addr":"127.0.0.1:45431"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.172742268Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot","remote_addr":"127.0.0.1:54209"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.172783304Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:63226","component":"dropshot","remote_addr":"127.0.0.1:49934"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.174717558Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_write","method":"POST","req_id":"6892ecaf-2aa0-45f9-aaf5-7dc549bacc4e","remote_addr":"127.0.0.1:35139","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3302,"response_code":"204"}
Sep 22 23:21:42.178 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.178 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.179 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.182 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.182 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.182 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.182492351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:33810","component":"dropshot","remote_addr":"127.0.0.1:40528"}
{"msg":"no entry exists for volume 715e89c9-d21c-47a9-83e7-53d748a8ba52, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183121664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18319643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183268009Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183307035Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: fb7bddce-e41d-4045-a067-a43fc478d6a6","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183351199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible fb7bddce-e41d-4045-a067-a43fc478d6a6 has session id: d07d0fc9-f7ce-4e03-abe9-08b4f5157d34","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183393284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:42682","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183429036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:61724","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183475801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:47055","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183582022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183645938Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183694961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183733091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 715e89c9-d21c-47a9-83e7-53d748a8ba52 constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.183860212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"fb7bddce-e41d-4045-a067-a43fc478d6a6 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18409566Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.184 INFO accepted connection from 127.0.0.1:51117, task: main
Sep 22 23:21:42.184 INFO accepted connection from 127.0.0.1:58664, task: main
Sep 22 23:21:42.184 INFO accepted connection from 127.0.0.1:48821, task: main
{"msg":"[0] fb7bddce-e41d-4045-a067-a43fc478d6a6 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.184455929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:42682 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18450747Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] fb7bddce-e41d-4045-a067-a43fc478d6a6 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.184550347Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:61724 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.184589832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] fb7bddce-e41d-4045-a067-a43fc478d6a6 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.184626348Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:47055 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.184664081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.184 INFO Connection request from fb7bddce-e41d-4045-a067-a43fc478d6a6 with version 4, task: proc
Sep 22 23:21:42.184 INFO upstairs UpstairsConnection { upstairs_id: fb7bddce-e41d-4045-a067-a43fc478d6a6, session_id: 18c3a639-2993-44be-a553-3c3d4014cfe6, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.184 INFO Connection request from fb7bddce-e41d-4045-a067-a43fc478d6a6 with version 4, task: proc
Sep 22 23:21:42.184 INFO upstairs UpstairsConnection { upstairs_id: fb7bddce-e41d-4045-a067-a43fc478d6a6, session_id: 18c3a639-2993-44be-a553-3c3d4014cfe6, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.184 INFO Connection request from fb7bddce-e41d-4045-a067-a43fc478d6a6 with version 4, task: proc
Sep 22 23:21:42.184 INFO upstairs UpstairsConnection { upstairs_id: fb7bddce-e41d-4045-a067-a43fc478d6a6, session_id: 18c3a639-2993-44be-a553-3c3d4014cfe6, gen: 1 } connected, version 4, task: proc
{"msg":"[0] fb7bddce-e41d-4045-a067-a43fc478d6a6 (18c3a639-2993-44be-a553-3c3d4014cfe6) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18509187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185141874Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 18c3a639-2993-44be-a553-3c3d4014cfe6","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185184195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] fb7bddce-e41d-4045-a067-a43fc478d6a6 (18c3a639-2993-44be-a553-3c3d4014cfe6) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185224295Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.185 INFO UpstairsConnection { upstairs_id: fb7bddce-e41d-4045-a067-a43fc478d6a6, session_id: 18c3a639-2993-44be-a553-3c3d4014cfe6, gen: 1 } is now active (read-write)
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185297913Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 18c3a639-2993-44be-a553-3c3d4014cfe6","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185337409Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.185 INFO UpstairsConnection { upstairs_id: fb7bddce-e41d-4045-a067-a43fc478d6a6, session_id: 18c3a639-2993-44be-a553-3c3d4014cfe6, gen: 1 } is now active (read-write)
{"msg":"[2] fb7bddce-e41d-4045-a067-a43fc478d6a6 (18c3a639-2993-44be-a553-3c3d4014cfe6) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185377042Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185416814Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.185 INFO UpstairsConnection { upstairs_id: fb7bddce-e41d-4045-a067-a43fc478d6a6, session_id: 18c3a639-2993-44be-a553-3c3d4014cfe6, gen: 1 } is now active (read-write)
{"msg":"[2] client is_active_req TRUE, promote! session 18c3a639-2993-44be-a553-3c3d4014cfe6","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185456925Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] downstairs client at 127.0.0.1:42682 has UUID 981e485a-3d34-445d-9adb-f9e13ff6f329","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185815451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 981e485a-3d34-445d-9adb-f9e13ff6f329, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185872656Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"fb7bddce-e41d-4045-a067-a43fc478d6a6 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185911225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:61724 has UUID c77d45f9-2dd1-4698-958d-b4ab9ce4ffee","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18594859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: c77d45f9-2dd1-4698-958d-b4ab9ce4ffee, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.185992344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"fb7bddce-e41d-4045-a067-a43fc478d6a6 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18602964Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:47055 has UUID 300935ac-634a-4338-b182-cfe66d8d8948","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.186070774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.186 INFO Current flush_numbers [0..12]: [0, 0]
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 300935ac-634a-4338-b182-cfe66d8d8948, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.186125321Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"fb7bddce-e41d-4045-a067-a43fc478d6a6 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.186164338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.186 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.186 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.186 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.186 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.186 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] fb7bddce-e41d-4045-a067-a43fc478d6a6 (18c3a639-2993-44be-a553-3c3d4014cfe6) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.186980976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187030377Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.187066713Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18710179Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] fb7bddce-e41d-4045-a067-a43fc478d6a6 (18c3a639-2993-44be-a553-3c3d4014cfe6) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187141813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187177507Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.187212744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187253797Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] fb7bddce-e41d-4045-a067-a43fc478d6a6 (18c3a639-2993-44be-a553-3c3d4014cfe6) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187296859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187332742Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.18736763Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187355824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_write","method":"POST","req_id":"ec612fd3-8be0-4cf9-ba1a-a365f863a0fb","remote_addr":"127.0.0.1:60149","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":12066,"response_code":"204"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187402187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:42682 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187459627Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187451111Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_write","method":"POST","req_id":"a8175294-6798-4939-95a3-c489b8ea8777","remote_addr":"127.0.0.1:62526","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":11958,"response_code":"204"}
{"msg":"fb7bddce-e41d-4045-a067-a43fc478d6a6 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187502467Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187543257Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187527422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_write","method":"POST","req_id":"3fdc08ef-c794-4d68-80e5-2f48fde6eea3","remote_addr":"127.0.0.1:63083","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":11834,"response_code":"204"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187579146Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187614206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18760078Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_write","method":"POST","req_id":"725d82f8-1008-4758-aea4-f4dfdc20eaa1","remote_addr":"127.0.0.1:46607","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":11693,"response_code":"204"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187648887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187683757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187673401Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_write","method":"POST","req_id":"e76c3b6d-894f-4b35-bfe7-c38f50c55fa0","remote_addr":"127.0.0.1:34605","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":11568,"response_code":"204"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187718382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187753133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187747326Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_write","method":"POST","req_id":"98a63973-7c40-48cb-80ed-273c32854cb5","remote_addr":"127.0.0.1:64643","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":11427,"response_code":"204"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18779768Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187837591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187819055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_write","method":"POST","req_id":"35b96d57-f7ae-4ed6-acc2-b15ef0a2c492","remote_addr":"127.0.0.1:45431","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":11273,"response_code":"204"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18787247Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187907905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187890343Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_write","method":"POST","req_id":"0438b2f4-a47b-4b9b-8901-7025dcbcdae8","remote_addr":"127.0.0.1:54209","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":11116,"response_code":"204"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187941851Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187976759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.187961076Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_write","method":"POST","req_id":"6517deb5-b551-4e0c-b9cf-cca9a7c97eae","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":10989,"response_code":"204"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188011613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188047757Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188087606Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"fb7bddce-e41d-4045-a067-a43fc478d6a6 is now active with session: 18c3a639-2993-44be-a553-3c3d4014cfe6","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188127483Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.188 DEBG Write     :1001 deps:[] res:true
{"msg":"fb7bddce-e41d-4045-a067-a43fc478d6a6 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.1881643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188254492Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188299847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:61724 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.18834064Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.188 DEBG Write     :1002 deps:[] res:true
{"msg":"fb7bddce-e41d-4045-a067-a43fc478d6a6 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188380344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188415798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:47055 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.1884515Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"fb7bddce-e41d-4045-a067-a43fc478d6a6 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188487464Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188521897Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188556864Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.188 DEBG Write     :1003 deps:[] res:true
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188592772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188633237Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188669756Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188704712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188738989Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188773741Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.188 DEBG Write     :1004 deps:[] res:true
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188808707Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188843438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 715e89c9-d21c-47a9-83e7-53d748a8ba52 activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188881496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 715e89c9-d21c-47a9-83e7-53d748a8ba52 constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188928471Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.188964712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/715e89c9-d21c-47a9-83e7-53d748a8ba52","method":"POST","req_id":"5d552db6-e614-4568-880f-9486fd8ee34d","remote_addr":"127.0.0.1:40528","local_addr":"127.0.0.1:33810","component":"dropshot","latency_us":4874,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.189030022Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/715e89c9-d21c-47a9-83e7-53d748a8ba52/validate","method":"POST","req_id":"62cda30e-1500-4fa5-8745-9d8a4919e931","remote_addr":"127.0.0.1:40528","local_addr":"127.0.0.1:33810","component":"dropshot","latency_us":237,"response_code":"200"}
Sep 22 23:21:42.189 DEBG Write     :1005 deps:[] res:true
Sep 22 23:21:42.189 DEBG Write     :1006 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.189351488Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/6a3c7f62-15b7-45cc-a4cb-0883cbca910e/is_finished","method":"GET","req_id":"85c1eddb-a341-49f3-aeb1-3b39e9fbd3b8","remote_addr":"127.0.0.1:40528","local_addr":"127.0.0.1:33810","component":"dropshot","latency_us":263,"response_code":"200"}
Sep 22 23:21:42.189 DEBG Write     :1007 deps:[] res:true
Sep 22 23:21:42.189 DEBG Write     :1008 deps:[] res:true
{"msg":"job 6a3c7f62-15b7-45cc-a4cb-0883cbca910e failed with size to validate 100 not divisible by block size 512!","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:42.189952308Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.189 DEBG Write     :1009 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.190000954Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/6a3c7f62-15b7-45cc-a4cb-0883cbca910e/ok","method":"GET","req_id":"cf48edb4-754a-4bb7-b4a1-fc11c9b7ce6a","remote_addr":"127.0.0.1:40528","local_addr":"127.0.0.1:33810","component":"dropshot","latency_us":207,"response_code":"200"}
{"msg":"detach removing entry for volume 715e89c9-d21c-47a9-83e7-53d748a8ba52","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.190582695Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume 715e89c9-d21c-47a9-83e7-53d748a8ba52","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.190632292Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.191 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
Sep 22 23:21:42.191 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
Sep 22 23:21:42.191 DEBG Flush     :1000 extent_limit None deps:[] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.191589318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"fb7bddce-e41d-4045-a067-a43fc478d6a6 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.191641458Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.191791595Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/715e89c9-d21c-47a9-83e7-53d748a8ba52","method":"DELETE","req_id":"189a4809-c0c9-496a-8105-12b798a256ca","remote_addr":"127.0.0.1:40528","local_addr":"127.0.0.1:33810","component":"dropshot","latency_us":1309,"response_code":"204"}
Sep 22 23:21:42.193 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:42.193 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:42.193 DEBG Write     :1003 deps:[] res:true
Sep 22 23:21:42.193 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:42.194 DEBG Write     :1005 deps:[] res:true
Sep 22 23:21:42.194 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:42.194 DEBG Write     :1007 deps:[] res:true
Sep 22 23:21:42.195 DEBG Write     :1008 deps:[] res:true
Sep 22 23:21:42.195 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:42.195 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:42.195 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:42.196 DEBG Write     :1003 deps:[] res:true
test test::test_pantry_validate_fail ... ok
Sep 22 23:21:42.196 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:42.196 INFO test_volume_replace of a volume
Sep 22 23:21:42.196 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.196 INFO Created new region file "/tmp/downstairs-KHkahjo0/region.json"
Sep 22 23:21:42.196 DEBG Write     :1005 deps:[] res:true
Sep 22 23:21:42.197 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:42.197 DEBG Write     :1007 deps:[] res:true
Sep 22 23:21:42.197 DEBG Write     :1008 deps:[] res:true
Sep 22 23:21:42.197 DEBG IO Write 1000 has deps []
Sep 22 23:21:42.197 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:42.197 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:21:42.198 DEBG up_ds_listen was notified
Sep 22 23:21:42.198 DEBG up_ds_listen process 1000
Sep 22 23:21:42.198 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:42.198 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:42.199 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.199 INFO Opened existing region file "/tmp/downstairs-KHkahjo0/region.json"
Sep 22 23:21:42.199 INFO Database read version 1
Sep 22 23:21:42.199 INFO Database write version 1
Sep 22 23:21:42.200 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:21:42.200 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:21:42.200 INFO UUID: 502a3035-412d-4e19-aa17-914c9ad2cb18
Sep 22 23:21:42.200 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.200 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:21:42.200 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.200 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.200 INFO Using address: 127.0.0.1:54814, task: main
Sep 22 23:21:42.201 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.201 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:48782, task: repair
Sep 22 23:21:42.201 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:48782, task: repair
Sep 22 23:21:42.201 INFO listening, local_addr: 127.0.0.1:48782, task: repair
Sep 22 23:21:42.201 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:48782, task: repair
Sep 22 23:21:42.201 INFO Using repair address: 127.0.0.1:48782, task: main
Sep 22 23:21:42.201 INFO No SSL acceptor configured, task: main
Sep 22 23:21:42.201 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.201 INFO Created new region file "/tmp/downstairs-PQIky7CE/region.json"
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.202862753Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_read","method":"POST","req_id":"4a927f49-cc52-4f9b-a198-e5a53914618d","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3736,"response_code":"200"}
Sep 22 23:21:42.204 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:21:42.204 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.204 INFO Opened existing region file "/tmp/downstairs-PQIky7CE/region.json"
Sep 22 23:21:42.204 INFO Database read version 1
Sep 22 23:21:42.204 INFO Database write version 1
Sep 22 23:21:42.205 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:21:42.205 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:21:42.205 INFO UUID: e3e919ab-aa67-4c3a-95f0-261ab63bbc0d
Sep 22 23:21:42.205 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.205 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.205 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.205 INFO Using address: 127.0.0.1:38745, task: main
Sep 22 23:21:42.206 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.206 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:34887, task: repair
Sep 22 23:21:42.206 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:34887, task: repair
Sep 22 23:21:42.206 INFO listening, local_addr: 127.0.0.1:34887, task: repair
Sep 22 23:21:42.206 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:34887, task: repair
Sep 22 23:21:42.206 INFO Using repair address: 127.0.0.1:34887, task: main
Sep 22 23:21:42.206 INFO No SSL acceptor configured, task: main
Sep 22 23:21:42.206 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.206 INFO Created new region file "/tmp/downstairs-1gPjNQ1p/region.json"
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.207178579Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_read","method":"POST","req_id":"35d58403-e2df-4252-9c72-95bab026aa9d","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3543,"response_code":"200"}
Sep 22 23:21:42.209 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:21:42.209 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:21:42.209 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:21:42.209 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.210 INFO Opened existing region file "/tmp/downstairs-1gPjNQ1p/region.json"
Sep 22 23:21:42.210 INFO Database read version 1
Sep 22 23:21:42.210 INFO Database write version 1
Sep 22 23:21:42.210 INFO UUID: 893fcb30-a534-4c9f-9b0b-98301d815098
Sep 22 23:21:42.210 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.210 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.210 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.211 INFO Using address: 127.0.0.1:41243, task: main
Sep 22 23:21:42.211 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.211 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:58505, task: repair
Sep 22 23:21:42.211 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:58505, task: repair
Sep 22 23:21:42.211 INFO listening, local_addr: 127.0.0.1:58505, task: repair
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.211609698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_read","method":"POST","req_id":"f593d9ca-b621-414d-9f96-7e3a565f6e48","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3682,"response_code":"200"}
Sep 22 23:21:42.211 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:58505, task: repair
Sep 22 23:21:42.211 INFO Using repair address: 127.0.0.1:58505, task: main
Sep 22 23:21:42.211 INFO No SSL acceptor configured, task: main
Sep 22 23:21:42.212 INFO Upstairs starts
Sep 22 23:21:42.212 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:42.212 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:42.212 INFO Crucible stats registered with UUID: ad483994-9168-4721-bea0-26e15aed46eb
Sep 22 23:21:42.212 INFO Crucible ad483994-9168-4721-bea0-26e15aed46eb has session id: 7829b344-35e9-4075-8db0-a4e3ccd0147f
Sep 22 23:21:42.212 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.212 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.212 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.212 INFO [0] connecting to 127.0.0.1:54814, looper: 0
Sep 22 23:21:42.212 INFO [1] connecting to 127.0.0.1:38745, looper: 1
Sep 22 23:21:42.212 INFO [2] connecting to 127.0.0.1:41243, looper: 2
Sep 22 23:21:42.212 INFO up_listen starts, task: up_listen
Sep 22 23:21:42.212 INFO Wait for all three downstairs to come online
Sep 22 23:21:42.213 INFO Flush timeout: 0.5
Sep 22 23:21:42.213 INFO [0] ad483994-9168-4721-bea0-26e15aed46eb looper connected, looper: 0
Sep 22 23:21:42.213 INFO [0] Proc runs for 127.0.0.1:54814 in state New
Sep 22 23:21:42.213 INFO accepted connection from 127.0.0.1:55920, task: main
Sep 22 23:21:42.213 INFO [1] ad483994-9168-4721-bea0-26e15aed46eb looper connected, looper: 1
Sep 22 23:21:42.213 INFO [1] Proc runs for 127.0.0.1:38745 in state New
Sep 22 23:21:42.213 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:21:42.213 INFO accepted connection from 127.0.0.1:37381, task: main
Sep 22 23:21:42.213 INFO [2] ad483994-9168-4721-bea0-26e15aed46eb looper connected, looper: 2
Sep 22 23:21:42.213 INFO [2] Proc runs for 127.0.0.1:41243 in state New
Sep 22 23:21:42.213 INFO accepted connection from 127.0.0.1:37150, task: main
Sep 22 23:21:42.213 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:21:42.213 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:21:42.213 INFO Connection request from ad483994-9168-4721-bea0-26e15aed46eb with version 4, task: proc
Sep 22 23:21:42.213 INFO upstairs UpstairsConnection { upstairs_id: ad483994-9168-4721-bea0-26e15aed46eb, session_id: d1488151-3064-4eea-a301-ac082b13982b, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:42.214 INFO Connection request from ad483994-9168-4721-bea0-26e15aed46eb with version 4, task: proc
Sep 22 23:21:42.214 INFO upstairs UpstairsConnection { upstairs_id: ad483994-9168-4721-bea0-26e15aed46eb, session_id: d1488151-3064-4eea-a301-ac082b13982b, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:42.214 INFO Connection request from ad483994-9168-4721-bea0-26e15aed46eb with version 4, task: proc
Sep 22 23:21:42.214 INFO upstairs UpstairsConnection { upstairs_id: ad483994-9168-4721-bea0-26e15aed46eb, session_id: d1488151-3064-4eea-a301-ac082b13982b, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:42.214 INFO [0] ad483994-9168-4721-bea0-26e15aed46eb (d1488151-3064-4eea-a301-ac082b13982b) New New New ds_transition to WaitActive
Sep 22 23:21:42.214 INFO [0] Transition from New to WaitActive
Sep 22 23:21:42.214 INFO [1] ad483994-9168-4721-bea0-26e15aed46eb (d1488151-3064-4eea-a301-ac082b13982b) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:42.214 INFO [1] Transition from New to WaitActive
Sep 22 23:21:42.214 INFO [2] ad483994-9168-4721-bea0-26e15aed46eb (d1488151-3064-4eea-a301-ac082b13982b) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:42.214 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:21:42.214 INFO ad483994-9168-4721-bea0-26e15aed46eb active request set
Sep 22 23:21:42.214 INFO [0] received activate with gen 2
Sep 22 23:21:42.214 INFO [0] client got ds_active_rx, promote! session d1488151-3064-4eea-a301-ac082b13982b
Sep 22 23:21:42.214 INFO [1] received activate with gen 2
Sep 22 23:21:42.214 INFO [1] client got ds_active_rx, promote! session d1488151-3064-4eea-a301-ac082b13982b
Sep 22 23:21:42.214 INFO [2] received activate with gen 2
Sep 22 23:21:42.214 INFO [2] client got ds_active_rx, promote! session d1488151-3064-4eea-a301-ac082b13982b
Sep 22 23:21:42.215 INFO UpstairsConnection { upstairs_id: ad483994-9168-4721-bea0-26e15aed46eb, session_id: d1488151-3064-4eea-a301-ac082b13982b, gen: 2 } is now active (read-write)
Sep 22 23:21:42.215 INFO UpstairsConnection { upstairs_id: ad483994-9168-4721-bea0-26e15aed46eb, session_id: d1488151-3064-4eea-a301-ac082b13982b, gen: 2 } is now active (read-write)
Sep 22 23:21:42.215 INFO UpstairsConnection { upstairs_id: ad483994-9168-4721-bea0-26e15aed46eb, session_id: d1488151-3064-4eea-a301-ac082b13982b, gen: 2 } is now active (read-write)
Sep 22 23:21:42.215 INFO [0] downstairs client at 127.0.0.1:54814 has UUID 502a3035-412d-4e19-aa17-914c9ad2cb18
Sep 22 23:21:42.215 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 502a3035-412d-4e19-aa17-914c9ad2cb18, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:42.215 INFO ad483994-9168-4721-bea0-26e15aed46eb WaitActive WaitActive WaitActive
Sep 22 23:21:42.215 INFO [1] downstairs client at 127.0.0.1:38745 has UUID e3e919ab-aa67-4c3a-95f0-261ab63bbc0d
Sep 22 23:21:42.215 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: e3e919ab-aa67-4c3a-95f0-261ab63bbc0d, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.215951129Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_read","method":"POST","req_id":"54450ad8-664d-442c-b8d7-98e498a485d4","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3592,"response_code":"200"}
Sep 22 23:21:42.216 INFO ad483994-9168-4721-bea0-26e15aed46eb WaitActive WaitActive WaitActive
Sep 22 23:21:42.216 INFO [2] downstairs client at 127.0.0.1:41243 has UUID 893fcb30-a534-4c9f-9b0b-98301d815098
Sep 22 23:21:42.216 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 893fcb30-a534-4c9f-9b0b-98301d815098, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:42.216 INFO ad483994-9168-4721-bea0-26e15aed46eb WaitActive WaitActive WaitActive
Sep 22 23:21:42.216 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.216 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.216 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.217 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.217 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.217 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.217 INFO [0] ad483994-9168-4721-bea0-26e15aed46eb (d1488151-3064-4eea-a301-ac082b13982b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:42.217 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:42.217 WARN [0] new RM replaced this: None
Sep 22 23:21:42.217 INFO [0] Starts reconcile loop
Sep 22 23:21:42.217 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:21:42.217 INFO [1] ad483994-9168-4721-bea0-26e15aed46eb (d1488151-3064-4eea-a301-ac082b13982b) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:42.217 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:42.217 WARN [1] new RM replaced this: None
Sep 22 23:21:42.217 INFO [1] Starts reconcile loop
Sep 22 23:21:42.218 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:21:42.218 INFO [2] ad483994-9168-4721-bea0-26e15aed46eb (d1488151-3064-4eea-a301-ac082b13982b) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:42.218 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:42.218 WARN [2] new RM replaced this: None
Sep 22 23:21:42.218 INFO [2] Starts reconcile loop
Sep 22 23:21:42.218 INFO [0] 127.0.0.1:54814 task reports connection:true
Sep 22 23:21:42.218 INFO ad483994-9168-4721-bea0-26e15aed46eb WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:42.218 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:21:42.218 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:42.218 INFO [0]R  generation: [0, 0]
Sep 22 23:21:42.218 INFO [0]R  dirty: [false, false]
Sep 22 23:21:42.218 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:42.218 INFO [1]R  generation: [0, 0]
Sep 22 23:21:42.218 INFO [1]R  dirty: [false, false]
Sep 22 23:21:42.218 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:42.218 INFO [2]R  generation: [0, 0]
Sep 22 23:21:42.218 INFO [2]R  dirty: [false, false]
Sep 22 23:21:42.218 INFO Max found gen is 1
Sep 22 23:21:42.218 INFO Generation requested: 2 >= found:1
Sep 22 23:21:42.218 INFO Next flush: 1
Sep 22 23:21:42.218 INFO All extents match
Sep 22 23:21:42.218 INFO No downstairs repair required
Sep 22 23:21:42.218 INFO No initial repair work was required
Sep 22 23:21:42.218 INFO Set Downstairs and Upstairs active
Sep 22 23:21:42.218 INFO ad483994-9168-4721-bea0-26e15aed46eb is now active with session: d1488151-3064-4eea-a301-ac082b13982b
Sep 22 23:21:42.218 INFO ad483994-9168-4721-bea0-26e15aed46eb Set Active after no repair
Sep 22 23:21:42.218 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:42.218 INFO Set check for repair
Sep 22 23:21:42.218 INFO [1] 127.0.0.1:38745 task reports connection:true
Sep 22 23:21:42.218 INFO ad483994-9168-4721-bea0-26e15aed46eb Active Active Active
Sep 22 23:21:42.218 INFO Set check for repair
Sep 22 23:21:42.218 INFO [2] 127.0.0.1:41243 task reports connection:true
Sep 22 23:21:42.218 INFO ad483994-9168-4721-bea0-26e15aed46eb Active Active Active
Sep 22 23:21:42.218 INFO Set check for repair
Sep 22 23:21:42.218 INFO [0] received reconcile message
Sep 22 23:21:42.218 INFO [0] All repairs completed, exit
Sep 22 23:21:42.218 INFO [0] Starts cmd_loop
Sep 22 23:21:42.218 INFO [1] received reconcile message
Sep 22 23:21:42.218 INFO [1] All repairs completed, exit
Sep 22 23:21:42.219 INFO [1] Starts cmd_loop
Sep 22 23:21:42.219 INFO [2] received reconcile message
Sep 22 23:21:42.219 INFO [2] All repairs completed, exit
Sep 22 23:21:42.219 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.22016787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_read","method":"POST","req_id":"d097b876-3f26-4331-853c-4e48326b1ca4","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3488,"response_code":"200"}
Sep 22 23:21:42.222 DEBG Read      :1015 deps:[JobId(1005)] res:true
Sep 22 23:21:42.222 DEBG IO Write 1000 has deps []
Sep 22 23:21:42.222 DEBG Read      :1015 deps:[JobId(1005)] res:true
Sep 22 23:21:42.222 DEBG up_ds_listen was notified
Sep 22 23:21:42.222 DEBG up_ds_listen process 1000
Sep 22 23:21:42.222 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:42.222 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:42.222 DEBG Read      :1015 deps:[JobId(1005)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.224435202Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_read","method":"POST","req_id":"a1c2f725-bacf-406d-b89f-906c252fa284","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3551,"response_code":"200"}
Sep 22 23:21:42.225 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.226 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:21:42.226 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:21:42.226 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.226 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:21:42.227 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.227 DEBG IO Read  1001 has deps [JobId(1000)]
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.228634053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_read","method":"POST","req_id":"82713f9b-af71-4ed9-989c-1283c6c76fe1","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3481,"response_code":"200"}
Sep 22 23:21:42.228 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:42.229 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:42.229 DEBG Read      :1001 deps:[JobId(1000)] res:true
Sep 22 23:21:42.230 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:21:42.230 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:21:42.230 DEBG Read      :1017 deps:[JobId(1007)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.2327593Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_read","method":"POST","req_id":"a1461ff2-a02c-4522-8ffe-252fd394f1c8","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3393,"response_code":"200"}
Sep 22 23:21:42.234 DEBG [0] Read AckReady 1001, : downstairs
Sep 22 23:21:42.234 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:21:42.234 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:21:42.234 DEBG Read      :1018 deps:[JobId(1008)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.236827726Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_read","method":"POST","req_id":"235f255c-5c74-44be-b926-e163ec05c50c","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3362,"response_code":"200"}
Sep 22 23:21:42.236 DEBG [1] Read already AckReady 1001, : downstairs
Sep 22 23:21:42.238 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:21:42.238 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:21:42.238 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:21:42.239 DEBG [2] Read already AckReady 1001, : downstairs
Sep 22 23:21:42.239 DEBG up_ds_listen was notified
Sep 22 23:21:42.239 DEBG up_ds_listen process 1001
Sep 22 23:21:42.239 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:42.240 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:42.240 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.240 INFO Created new region file "/tmp/downstairs-kGoDqY3e/region.json"
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.240913801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/bulk_read","method":"POST","req_id":"4c107f3e-b314-4820-b08c-447a8b7ab499","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3370,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.242618396Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434/validate","method":"POST","req_id":"25fd763b-07c8-431b-bbd6-fe4175534a9e","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":321,"response_code":"200"}
Sep 22 23:21:42.244 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.244 INFO Opened existing region file "/tmp/downstairs-kGoDqY3e/region.json"
Sep 22 23:21:42.244 INFO Database read version 1
Sep 22 23:21:42.244 INFO Database write version 1
Sep 22 23:21:42.244 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:21:42.244 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:21:42.244 INFO UUID: 9425657f-af69-47cb-8591-774480ba4038
Sep 22 23:21:42.244 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.245 DEBG Read      :1020 deps:[JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:21:42.245 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.245 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.245 INFO Using address: 127.0.0.1:64531, task: main
Sep 22 23:21:42.245 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.245 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:35080, task: repair
Sep 22 23:21:42.245 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:35080, task: repair
Sep 22 23:21:42.245 INFO listening, local_addr: 127.0.0.1:35080, task: repair
Sep 22 23:21:42.245 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:35080, task: repair
Sep 22 23:21:42.245 INFO Using repair address: 127.0.0.1:35080, task: main
Sep 22 23:21:42.246 INFO No SSL acceptor configured, task: main
Sep 22 23:21:42.246 INFO A New downstairs: 127.0.0.1:64531
Sep 22 23:21:42.246 INFO Old ops target: [127.0.0.1:54814, 127.0.0.1:38745, 127.0.0.1:41243]
Sep 22 23:21:42.246 INFO New ops target: [127.0.0.1:64531, 127.0.0.1:38745, 127.0.0.1:41243]
Sep 22 23:21:42.246 INFO Replace VCR now: Volume { id: f58b3665-bdc0-4be9-8cf4-eefa2c09c427, block_size: 512, sub_volumes: [Region { block_size: 512, blocks_per_extent: 5, extent_count: 2, opts: CrucibleOpts { id: ad483994-9168-4721-bea0-26e15aed46eb, target: [127.0.0.1:64531, 127.0.0.1:38745, 127.0.0.1:41243], lossy: false, flush_timeout: None, key: Some("dEINxGY0b3GK6upU+NElp3abG+t59SMWQZT7RPtIozo="), cert_pem: None, key_pem: None, root_cert_pem: None, control: None, read_only: false }, gen: 3 }], read_only_parent: None }
Sep 22 23:21:42.246 INFO Volume f58b3665-bdc0-4be9-8cf4-eefa2c09c427, OK to replace: 127.0.0.1:54814 with 127.0.0.1:64531
Sep 22 23:21:42.246 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.246 WARN f58b3665-bdc0-4be9-8cf4-eefa2c09c427 request to replace downstairs 127.0.0.1:54814 with 127.0.0.1:64531
Sep 22 23:21:42.246 INFO f58b3665-bdc0-4be9-8cf4-eefa2c09c427 found old target: 127.0.0.1:54814 at 0
Sep 22 23:21:42.246 INFO f58b3665-bdc0-4be9-8cf4-eefa2c09c427 replacing old: 127.0.0.1:54814 at 0
Sep 22 23:21:42.246 INFO [0] client skip 2 in process jobs because fault, : downstairs
Sep 22 23:21:42.246 INFO [0] changed 0 jobs to fault skipped, : downstairs
Sep 22 23:21:42.246 INFO [0] ad483994-9168-4721-bea0-26e15aed46eb (d1488151-3064-4eea-a301-ac082b13982b) Active Active Active ds_transition to Replacing
Sep 22 23:21:42.246 INFO [0] Transition from Active to Replacing
Sep 22 23:21:42.246 INFO Replace downstairs underway for f58b3665-bdc0-4be9-8cf4-eefa2c09c427
Sep 22 23:21:42.246 INFO send read now
Sep 22 23:21:42.246 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.246 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.246 DEBG IO Read  1002 has deps [JobId(1000)]
Sep 22 23:21:42.247 INFO listening on 127.0.0.1:0, task: main
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.247364974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot","remote_addr":"127.0.0.1:39338"}
Sep 22 23:21:42.247 DEBG Read      :1002 deps:[JobId(1000)] res:true
{"msg":"no entry exists for volume 52ed3973-f354-40ba-a6f9-b5522a98667e, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.248035119Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.248 DEBG Read      :1002 deps:[JobId(1000)] res:true
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.248284465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.248336648Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.248361824Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 81fb9c00-5d21-4891-b458-1cfd5eccef89","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.248385806Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible 81fb9c00-5d21-4891-b458-1cfd5eccef89 has session id: 4ce233b0-59d9-436c-93c7-f67280b0f01b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.248410426Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:49499","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.248565767Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:56926","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.248750792Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:46734","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.248890716Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.248967643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.249020802Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.249065659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 52ed3973-f354-40ba-a6f9-b5522a98667e constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.249256168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"81fb9c00-5d21-4891-b458-1cfd5eccef89 active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.249511465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.249 INFO accepted connection from 127.0.0.1:33803, task: main
Sep 22 23:21:42.249 INFO accepted connection from 127.0.0.1:56045, task: main
Sep 22 23:21:42.249 INFO accepted connection from 127.0.0.1:47897, task: main
{"msg":"[0] 81fb9c00-5d21-4891-b458-1cfd5eccef89 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.249840737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:49499 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.249897666Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 81fb9c00-5d21-4891-b458-1cfd5eccef89 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.249946676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:56926 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.249997034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 81fb9c00-5d21-4891-b458-1cfd5eccef89 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.250044533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:46734 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.250093034Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.250 INFO Connection request from 81fb9c00-5d21-4891-b458-1cfd5eccef89 with version 4, task: proc
Sep 22 23:21:42.250 INFO upstairs UpstairsConnection { upstairs_id: 81fb9c00-5d21-4891-b458-1cfd5eccef89, session_id: 09d4dd63-aea1-4a34-834b-c41ae295a586, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.250 INFO Connection request from 81fb9c00-5d21-4891-b458-1cfd5eccef89 with version 4, task: proc
Sep 22 23:21:42.250 INFO upstairs UpstairsConnection { upstairs_id: 81fb9c00-5d21-4891-b458-1cfd5eccef89, session_id: 09d4dd63-aea1-4a34-834b-c41ae295a586, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.250 INFO Connection request from 81fb9c00-5d21-4891-b458-1cfd5eccef89 with version 4, task: proc
Sep 22 23:21:42.250 INFO upstairs UpstairsConnection { upstairs_id: 81fb9c00-5d21-4891-b458-1cfd5eccef89, session_id: 09d4dd63-aea1-4a34-834b-c41ae295a586, gen: 1 } connected, version 4, task: proc
{"msg":"[0] 81fb9c00-5d21-4891-b458-1cfd5eccef89 (09d4dd63-aea1-4a34-834b-c41ae295a586) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.250571344Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.250626962Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] client is_active_req TRUE, promote! session 09d4dd63-aea1-4a34-834b-c41ae295a586","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.250677248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 81fb9c00-5d21-4891-b458-1cfd5eccef89 (09d4dd63-aea1-4a34-834b-c41ae295a586) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.250728366Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.250781956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client is_active_req TRUE, promote! session 09d4dd63-aea1-4a34-834b-c41ae295a586","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.25083204Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.250 INFO UpstairsConnection { upstairs_id: 81fb9c00-5d21-4891-b458-1cfd5eccef89, session_id: 09d4dd63-aea1-4a34-834b-c41ae295a586, gen: 1 } is now active (read-write)
{"msg":"[2] 81fb9c00-5d21-4891-b458-1cfd5eccef89 (09d4dd63-aea1-4a34-834b-c41ae295a586) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.250883195Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.250924142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] client is_active_req TRUE, promote! session 09d4dd63-aea1-4a34-834b-c41ae295a586","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.250971142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.250 INFO UpstairsConnection { upstairs_id: 81fb9c00-5d21-4891-b458-1cfd5eccef89, session_id: 09d4dd63-aea1-4a34-834b-c41ae295a586, gen: 1 } is now active (read-write)
Sep 22 23:21:42.251 INFO UpstairsConnection { upstairs_id: 81fb9c00-5d21-4891-b458-1cfd5eccef89, session_id: 09d4dd63-aea1-4a34-834b-c41ae295a586, gen: 1 } is now active (read-write)
{"msg":"[0] downstairs client at 127.0.0.1:49499 has UUID 101c5e08-1317-4b60-a031-62766e2cdb1f","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.251496657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 101c5e08-1317-4b60-a031-62766e2cdb1f, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.251559718Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"81fb9c00-5d21-4891-b458-1cfd5eccef89 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.251609438Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:56926 has UUID 44286653-4082-4ed7-a812-3555da359c78","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.251659135Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 44286653-4082-4ed7-a812-3555da359c78, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.251713645Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"81fb9c00-5d21-4891-b458-1cfd5eccef89 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.251762564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:46734 has UUID d4d95f2d-aa01-4319-8829-47ddcf12c9ca","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.251810905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: d4d95f2d-aa01-4319-8829-47ddcf12c9ca, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.251871811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.251 INFO Current flush_numbers [0..12]: [0, 0]
{"msg":"81fb9c00-5d21-4891-b458-1cfd5eccef89 WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.251924561Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.252 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.252 DEBG [1] Read AckReady 1002, : downstairs
Sep 22 23:21:42.252 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.252 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.252 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.252 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 81fb9c00-5d21-4891-b458-1cfd5eccef89 (09d4dd63-aea1-4a34-834b-c41ae295a586) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.252904256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.252960725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.253008383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253055961Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 81fb9c00-5d21-4891-b458-1cfd5eccef89 (09d4dd63-aea1-4a34-834b-c41ae295a586) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253107315Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253153825Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.253195148Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253237916Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 81fb9c00-5d21-4891-b458-1cfd5eccef89 (09d4dd63-aea1-4a34-834b-c41ae295a586) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253286053Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253331793Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:42.253376915Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253422381Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:49499 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253475811Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"81fb9c00-5d21-4891-b458-1cfd5eccef89 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253524031Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253570664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253615408Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253661176Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253707705Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253752227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253795882Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  flush_numbers: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253840628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  generation: [0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253883896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R  dirty: [false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253929436Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.253974564Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254018667Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254063048Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254107703Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254152154Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254196714Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254241133Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"81fb9c00-5d21-4891-b458-1cfd5eccef89 is now active with session: 09d4dd63-aea1-4a34-834b-c41ae295a586","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254289036Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"81fb9c00-5d21-4891-b458-1cfd5eccef89 Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254338751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254382382Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254426709Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:56926 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.25447073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"81fb9c00-5d21-4891-b458-1cfd5eccef89 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254515484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254558448Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:46734 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254602589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"81fb9c00-5d21-4891-b458-1cfd5eccef89 Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254647024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254690557Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.25473392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254777444Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254820058Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.25486296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254904774Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.254948959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.2549935Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.255033801Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.25507117Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 52ed3973-f354-40ba-a6f9-b5522a98667e activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.255112887Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.255 DEBG [2] Read already AckReady 1002, : downstairs
{"msg":"volume 52ed3973-f354-40ba-a6f9-b5522a98667e constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.255167008Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.255 DEBG up_ds_listen was notified
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.25521369Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e","method":"POST","req_id":"c4437a54-300e-4d5d-bfb5-4143cabd857d","remote_addr":"127.0.0.1:39338","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":6173,"response_code":"200"}
Sep 22 23:21:42.255 DEBG up_ds_listen process 1002
Sep 22 23:21:42.255 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:42.255 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.257481191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/c5c977cd-0026-4860-a0d3-20d2ec1cd1ae/is_finished","method":"GET","req_id":"a4534583-1590-40b0-bb5c-c799be7ac989","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":228,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.258361533Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/c5c977cd-0026-4860-a0d3-20d2ec1cd1ae/ok","method":"GET","req_id":"978d0e87-f6b1-4e30-a34e-e62b9a78943b","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":232,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.259079169Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot","remote_addr":"127.0.0.1:32779"}
{"msg":"detach removing entry for volume fc92b52b-99b5-4d79-ab62-675ee18fb434","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.25910338Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.259143296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot","remote_addr":"127.0.0.1:49612"}
{"msg":"detaching volume fc92b52b-99b5-4d79-ab62-675ee18fb434","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.259161359Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.259189906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot","remote_addr":"127.0.0.1:58198"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.259235585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot","remote_addr":"127.0.0.1:62417"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.259282049Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot","remote_addr":"127.0.0.1:49581"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.259326071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot","remote_addr":"127.0.0.1:46603"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.259378473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot","remote_addr":"127.0.0.1:47870"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.259434852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot","remote_addr":"127.0.0.1:33608"}
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.259480239Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:59371","component":"dropshot","remote_addr":"127.0.0.1:53624"}
test test::test_volume_replace_vcr ... ok
Sep 22 23:21:42.260 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.260 INFO Created new region file "/tmp/downstairs-suheTfUx/region.json"
Sep 22 23:21:42.261 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:42.261 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.261158759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_write","method":"POST","req_id":"fe332a10-b8c8-4539-a056-259538a35a19","remote_addr":"127.0.0.1:39338","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":2421,"response_code":"204"}
Sep 22 23:21:42.261 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.261931481Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"94d7a72a-673e-42de-882f-9dbeb845d1f4 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.261990044Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.262161778Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/fc92b52b-99b5-4d79-ab62-675ee18fb434","method":"DELETE","req_id":"3ae54a56-4edd-421c-bfd4-826ce7acddfa","remote_addr":"127.0.0.1:49934","local_addr":"127.0.0.1:63226","component":"dropshot","latency_us":3147,"response_code":"204"}
Sep 22 23:21:42.264 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.264 INFO Opened existing region file "/tmp/downstairs-suheTfUx/region.json"
Sep 22 23:21:42.264 INFO Database read version 1
Sep 22 23:21:42.264 INFO Database write version 1
Sep 22 23:21:42.264 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.265 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.265 INFO UUID: 80aebe51-0bf8-4d2d-b94d-436bb289b850
Sep 22 23:21:42.265 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.265 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.265 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.265 INFO Using address: 127.0.0.1:34016, task: main
Sep 22 23:21:42.265 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.265 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.265 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:49270, task: repair
Sep 22 23:21:42.265 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:49270, task: repair
Sep 22 23:21:42.265 INFO listening, local_addr: 127.0.0.1:49270, task: repair
Sep 22 23:21:42.266 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:49270, task: repair
Sep 22 23:21:42.266 INFO Using repair address: 127.0.0.1:49270, task: main
Sep 22 23:21:42.266 INFO No SSL acceptor configured, task: main
Sep 22 23:21:42.266 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.266 INFO Created new region file "/tmp/downstairs-VlIPI8qJ/region.json"
test test::test_pantry_validate ... ok
Sep 22 23:21:42.268 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.269 INFO Opened existing region file "/tmp/downstairs-VlIPI8qJ/region.json"
Sep 22 23:21:42.269 INFO Database read version 1
Sep 22 23:21:42.269 INFO Database write version 1
Sep 22 23:21:42.269 INFO UUID: 9b908e92-ae00-4395-8352-4c1a031a61cb
Sep 22 23:21:42.269 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.269 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.269 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.269 INFO Using address: 127.0.0.1:62390, task: main
Sep 22 23:21:42.269 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:42.270 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:44742, task: repair
Sep 22 23:21:42.270 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:44742, task: repair
Sep 22 23:21:42.270 INFO listening, local_addr: 127.0.0.1:44742, task: repair
Sep 22 23:21:42.270 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:44742, task: repair
Sep 22 23:21:42.270 INFO Using repair address: 127.0.0.1:44742, task: main
Sep 22 23:21:42.270 INFO No SSL acceptor configured, task: main
Sep 22 23:21:42.270 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.270 INFO Created new region file "/tmp/downstairs-lyKODt8n/region.json"
Sep 22 23:21:42.272 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:42.272 INFO Opened existing region file "/tmp/downstairs-lyKODt8n/region.json"
Sep 22 23:21:42.272 INFO Database read version 1
Sep 22 23:21:42.272 INFO Database write version 1
Sep 22 23:21:42.273 INFO UUID: 4e224a63-3b6f-46b4-89c9-d64f655331fa
Sep 22 23:21:42.273 INFO Blocks per extent:5 Total Extents: 2
Sep 22 23:21:42.273 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:42.273 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:42.273 INFO Using address: 127.0.0.1:47605, task: main
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.2737496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_write","method":"POST","req_id":"d0c86c51-59b9-4d14-9f60-3e59258f67dc","remote_addr":"127.0.0.1:32779","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":12421,"response_code":"204"}
Sep 22 23:21:42.273 INFO Repair listens on 127.0.0.1:0, task: repair
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.273815772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_write","method":"POST","req_id":"b5d82e11-bf71-4992-8b64-4b8e85ca0847","remote_addr":"127.0.0.1:49612","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":12382,"response_code":"204"}
Sep 22 23:21:42.273 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:55828, task: repair
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.27387639Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_write","method":"POST","req_id":"801c2da9-a72b-4238-b2b0-07d5797deb5b","remote_addr":"127.0.0.1:58198","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":12288,"response_code":"204"}
Sep 22 23:21:42.273 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:55828, task: repair
Sep 22 23:21:42.273 INFO listening, local_addr: 127.0.0.1:55828, task: repair
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.273987302Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_write","method":"POST","req_id":"a45c801c-f4ad-4d7c-ab36-5c872260b59d","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":12186,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.274037884Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_write","method":"POST","req_id":"6a75311c-647e-406e-a918-46574a94f203","remote_addr":"127.0.0.1:49581","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":12099,"response_code":"204"}
Sep 22 23:21:42.274 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:55828, task: repair
Sep 22 23:21:42.274 INFO Using repair address: 127.0.0.1:55828, task: main
Sep 22 23:21:42.274 INFO No SSL acceptor configured, task: main
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.274146655Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_write","method":"POST","req_id":"e22158a5-641d-4d28-9873-80d814883240","remote_addr":"127.0.0.1:46603","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":11973,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.274197168Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_write","method":"POST","req_id":"ca64d123-48cd-4ca1-b6e9-3330fc7ca647","remote_addr":"127.0.0.1:47870","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":11875,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.274310057Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_write","method":"POST","req_id":"1e1051d7-e07d-4538-a5ca-60bee59382bb","remote_addr":"127.0.0.1:33608","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":11772,"response_code":"204"}
Sep 22 23:21:42.274 INFO Upstairs starts
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.274363552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_write","method":"POST","req_id":"aa30de07-5767-4abd-9af3-aa6b3ec98516","remote_addr":"127.0.0.1:53624","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":11713,"response_code":"204"}
Sep 22 23:21:42.274 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:42.274 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:42.274 INFO Crucible stats registered with UUID: 799d475b-e46c-45c2-863b-4988622fc243
Sep 22 23:21:42.274 INFO Crucible 799d475b-e46c-45c2-863b-4988622fc243 has session id: 59749590-6fe4-4251-b14d-99efeb11148c
Sep 22 23:21:42.274 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.274 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.274 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:42.274 INFO [0] connecting to 127.0.0.1:34016, looper: 0
Sep 22 23:21:42.274 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:42.274 INFO [1] connecting to 127.0.0.1:62390, looper: 1
Sep 22 23:21:42.274 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:42.274 INFO [2] connecting to 127.0.0.1:47605, looper: 2
Sep 22 23:21:42.275 INFO up_listen starts, task: up_listen
Sep 22 23:21:42.275 INFO Wait for all three downstairs to come online
Sep 22 23:21:42.275 INFO Flush timeout: 0.5
Sep 22 23:21:42.275 INFO [0] 799d475b-e46c-45c2-863b-4988622fc243 looper connected, looper: 0
Sep 22 23:21:42.275 DEBG Write     :1003 deps:[] res:true
Sep 22 23:21:42.275 INFO [0] Proc runs for 127.0.0.1:34016 in state New
Sep 22 23:21:42.275 INFO [1] 799d475b-e46c-45c2-863b-4988622fc243 looper connected, looper: 1
Sep 22 23:21:42.275 INFO [1] Proc runs for 127.0.0.1:62390 in state New
Sep 22 23:21:42.275 INFO accepted connection from 127.0.0.1:36492, task: main
Sep 22 23:21:42.275 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:42.275 INFO accepted connection from 127.0.0.1:38651, task: main
Sep 22 23:21:42.275 INFO [2] 799d475b-e46c-45c2-863b-4988622fc243 looper connected, looper: 2
Sep 22 23:21:42.275 INFO [2] Proc runs for 127.0.0.1:47605 in state New
Sep 22 23:21:42.275 INFO accepted connection from 127.0.0.1:53191, task: main
Sep 22 23:21:42.275 DEBG Write     :1005 deps:[] res:true
Sep 22 23:21:42.275 INFO Connection request from 799d475b-e46c-45c2-863b-4988622fc243 with version 4, task: proc
Sep 22 23:21:42.275 INFO upstairs UpstairsConnection { upstairs_id: 799d475b-e46c-45c2-863b-4988622fc243, session_id: 6cb054b3-bde9-4bc7-a1bb-ff874c7b4437, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.275 INFO Connection request from 799d475b-e46c-45c2-863b-4988622fc243 with version 4, task: proc
Sep 22 23:21:42.275 INFO upstairs UpstairsConnection { upstairs_id: 799d475b-e46c-45c2-863b-4988622fc243, session_id: 6cb054b3-bde9-4bc7-a1bb-ff874c7b4437, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.275 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:42.276 INFO Connection request from 799d475b-e46c-45c2-863b-4988622fc243 with version 4, task: proc
Sep 22 23:21:42.276 INFO upstairs UpstairsConnection { upstairs_id: 799d475b-e46c-45c2-863b-4988622fc243, session_id: 6cb054b3-bde9-4bc7-a1bb-ff874c7b4437, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:42.276 DEBG Write     :1007 deps:[] res:true
Sep 22 23:21:42.276 INFO [0] 799d475b-e46c-45c2-863b-4988622fc243 (6cb054b3-bde9-4bc7-a1bb-ff874c7b4437) New New New ds_transition to WaitActive
Sep 22 23:21:42.276 INFO [0] Transition from New to WaitActive
Sep 22 23:21:42.276 INFO [1] 799d475b-e46c-45c2-863b-4988622fc243 (6cb054b3-bde9-4bc7-a1bb-ff874c7b4437) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:42.276 INFO [1] Transition from New to WaitActive
Sep 22 23:21:42.276 DEBG Write     :1008 deps:[] res:true
Sep 22 23:21:42.276 INFO [2] 799d475b-e46c-45c2-863b-4988622fc243 (6cb054b3-bde9-4bc7-a1bb-ff874c7b4437) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:42.276 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:21:42.276 INFO 799d475b-e46c-45c2-863b-4988622fc243 active request set
Sep 22 23:21:42.276 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:42.276 INFO [0] received activate with gen 1
Sep 22 23:21:42.276 INFO [0] client got ds_active_rx, promote! session 6cb054b3-bde9-4bc7-a1bb-ff874c7b4437
Sep 22 23:21:42.276 INFO [1] received activate with gen 1
Sep 22 23:21:42.276 INFO [1] client got ds_active_rx, promote! session 6cb054b3-bde9-4bc7-a1bb-ff874c7b4437
Sep 22 23:21:42.276 INFO [2] received activate with gen 1
Sep 22 23:21:42.276 INFO [2] client got ds_active_rx, promote! session 6cb054b3-bde9-4bc7-a1bb-ff874c7b4437
Sep 22 23:21:42.276 INFO UpstairsConnection { upstairs_id: 799d475b-e46c-45c2-863b-4988622fc243, session_id: 6cb054b3-bde9-4bc7-a1bb-ff874c7b4437, gen: 1 } is now active (read-write)
Sep 22 23:21:42.276 INFO UpstairsConnection { upstairs_id: 799d475b-e46c-45c2-863b-4988622fc243, session_id: 6cb054b3-bde9-4bc7-a1bb-ff874c7b4437, gen: 1 } is now active (read-write)
Sep 22 23:21:42.277 INFO UpstairsConnection { upstairs_id: 799d475b-e46c-45c2-863b-4988622fc243, session_id: 6cb054b3-bde9-4bc7-a1bb-ff874c7b4437, gen: 1 } is now active (read-write)
Sep 22 23:21:42.277 INFO [0] downstairs client at 127.0.0.1:34016 has UUID 80aebe51-0bf8-4d2d-b94d-436bb289b850
Sep 22 23:21:42.277 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 80aebe51-0bf8-4d2d-b94d-436bb289b850, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:42.277 INFO 799d475b-e46c-45c2-863b-4988622fc243 WaitActive WaitActive WaitActive
Sep 22 23:21:42.277 INFO [1] downstairs client at 127.0.0.1:62390 has UUID 9b908e92-ae00-4395-8352-4c1a031a61cb
Sep 22 23:21:42.277 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 9b908e92-ae00-4395-8352-4c1a031a61cb, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:42.277 INFO 799d475b-e46c-45c2-863b-4988622fc243 WaitActive WaitActive WaitActive
Sep 22 23:21:42.277 INFO [2] downstairs client at 127.0.0.1:47605 has UUID 4e224a63-3b6f-46b4-89c9-d64f655331fa
Sep 22 23:21:42.277 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 5, shift: 9 }, extent_count: 2, uuid: 4e224a63-3b6f-46b4-89c9-d64f655331fa, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:42.277 INFO 799d475b-e46c-45c2-863b-4988622fc243 WaitActive WaitActive WaitActive
Sep 22 23:21:42.277 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.277 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.278 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.278 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.278 INFO Current flush_numbers [0..12]: [0, 0]
Sep 22 23:21:42.278 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:42.278 INFO [0] 799d475b-e46c-45c2-863b-4988622fc243 (6cb054b3-bde9-4bc7-a1bb-ff874c7b4437) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:42.278 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:42.278 WARN [0] new RM replaced this: None
Sep 22 23:21:42.278 INFO [0] Starts reconcile loop
Sep 22 23:21:42.278 INFO [1] 799d475b-e46c-45c2-863b-4988622fc243 (6cb054b3-bde9-4bc7-a1bb-ff874c7b4437) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:42.278 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:42.278 WARN [1] new RM replaced this: None
Sep 22 23:21:42.278 INFO [1] Starts reconcile loop
Sep 22 23:21:42.278 INFO [2] 799d475b-e46c-45c2-863b-4988622fc243 (6cb054b3-bde9-4bc7-a1bb-ff874c7b4437) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:42.278 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:42.278 WARN [2] new RM replaced this: None
Sep 22 23:21:42.278 INFO [2] Starts reconcile loop
Sep 22 23:21:42.278 INFO [0] 127.0.0.1:34016 task reports connection:true
Sep 22 23:21:42.278 INFO 799d475b-e46c-45c2-863b-4988622fc243 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:42.278 INFO [0]R  flush_numbers: [0, 0]
Sep 22 23:21:42.278 INFO [0]R  generation: [0, 0]
Sep 22 23:21:42.278 INFO [0]R  dirty: [false, false]
Sep 22 23:21:42.279 INFO [1]R  flush_numbers: [0, 0]
Sep 22 23:21:42.279 INFO [1]R  generation: [0, 0]
Sep 22 23:21:42.279 INFO [1]R  dirty: [false, false]
Sep 22 23:21:42.279 INFO [2]R  flush_numbers: [0, 0]
Sep 22 23:21:42.279 INFO [2]R  generation: [0, 0]
Sep 22 23:21:42.279 INFO [2]R  dirty: [false, false]
Sep 22 23:21:42.279 INFO Max found gen is 1
Sep 22 23:21:42.279 INFO Generation requested: 1 >= found:1
Sep 22 23:21:42.279 INFO Next flush: 1
Sep 22 23:21:42.279 INFO All extents match
Sep 22 23:21:42.279 INFO No downstairs repair required
Sep 22 23:21:42.279 INFO No initial repair work was required
Sep 22 23:21:42.279 INFO Set Downstairs and Upstairs active
Sep 22 23:21:42.279 INFO 799d475b-e46c-45c2-863b-4988622fc243 is now active with session: 6cb054b3-bde9-4bc7-a1bb-ff874c7b4437
Sep 22 23:21:42.279 INFO 799d475b-e46c-45c2-863b-4988622fc243 Set Active after no repair
Sep 22 23:21:42.279 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:42.279 INFO Set check for repair
Sep 22 23:21:42.279 INFO [1] 127.0.0.1:62390 task reports connection:true
Sep 22 23:21:42.279 INFO 799d475b-e46c-45c2-863b-4988622fc243 Active Active Active
Sep 22 23:21:42.279 INFO Set check for repair
Sep 22 23:21:42.279 INFO [2] 127.0.0.1:47605 task reports connection:true
Sep 22 23:21:42.279 INFO 799d475b-e46c-45c2-863b-4988622fc243 Active Active Active
Sep 22 23:21:42.279 INFO Set check for repair
Sep 22 23:21:42.279 INFO [0] received reconcile message
Sep 22 23:21:42.279 INFO [0] All repairs completed, exit
Sep 22 23:21:42.279 INFO [0] Starts cmd_loop
Sep 22 23:21:42.279 INFO [1] received reconcile message
Sep 22 23:21:42.279 INFO [1] All repairs completed, exit
Sep 22 23:21:42.279 INFO [1] Starts cmd_loop
Sep 22 23:21:42.279 INFO [2] received reconcile message
Sep 22 23:21:42.279 INFO [2] All repairs completed, exit
Sep 22 23:21:42.279 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:42.279 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:42.280 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:42.280 DEBG Write     :1003 deps:[] res:true
Sep 22 23:21:42.280 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:42.281 DEBG Write     :1005 deps:[] res:true
Sep 22 23:21:42.281 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:42.281 DEBG Write     :1007 deps:[] res:true
test test::volume_zero_length_io ... ok
Sep 22 23:21:42.282 DEBG Write     :1008 deps:[] res:true
Sep 22 23:21:42.282 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:42.282 DEBG Write     :1001 deps:[] res:true
Sep 22 23:21:42.282 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:42.282 DEBG Write     :1003 deps:[] res:true
Sep 22 23:21:42.283 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:42.283 DEBG Write     :1005 deps:[] res:true
Sep 22 23:21:42.283 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:42.283 DEBG Write     :1007 deps:[] res:true
Sep 22 23:21:42.284 DEBG Write     :1008 deps:[] res:true
Sep 22 23:21:42.284 DEBG Write     :1009 deps:[] res:true
Sep 22 23:21:42.285 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:21:42.286 DEBG Read      :1010 deps:[JobId(1000)] res:true
Sep 22 23:21:42.286 DEBG Read      :1010 deps:[JobId(1000)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.287673358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_read","method":"POST","req_id":"53c06d21-f6d1-4def-a90f-5a404512ac37","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":2597,"response_code":"200"}
Sep 22 23:21:42.289 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:21:42.289 DEBG Read      :1011 deps:[JobId(1001)] res:true
Sep 22 23:21:42.289 DEBG Read      :1011 deps:[JobId(1001)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.290662107Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_read","method":"POST","req_id":"10700452-e31f-4920-960b-a702c1710188","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":2421,"response_code":"200"}
Sep 22 23:21:42.292 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:21:42.292 DEBG Read      :1012 deps:[JobId(1002)] res:true
Sep 22 23:21:42.292 DEBG Read      :1012 deps:[JobId(1002)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.29362454Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_read","method":"POST","req_id":"741197ec-27d2-4fa8-9e82-4b6e49eb346a","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":2413,"response_code":"200"}
Sep 22 23:21:42.294 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:21:42.295 DEBG Read      :1013 deps:[JobId(1003)] res:true
Sep 22 23:21:42.295 DEBG Read      :1013 deps:[JobId(1003)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.296574296Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_read","method":"POST","req_id":"0e0b4902-696e-4fd5-a4de-3dc810389cb2","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":2400,"response_code":"200"}
Sep 22 23:21:42.297 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:21:42.298 DEBG Read      :1014 deps:[JobId(1004)] res:true
Sep 22 23:21:42.298 DEBG Read      :1014 deps:[JobId(1004)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.299547106Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_read","method":"POST","req_id":"e7690cac-8556-46a5-88df-bb2c2c4e6a7b","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":2423,"response_code":"200"}
Sep 22 23:21:42.300 DEBG Read      :1015 deps:[JobId(1005)] res:true
Sep 22 23:21:42.301 DEBG Read      :1015 deps:[JobId(1005)] res:true
Sep 22 23:21:42.301 DEBG Read      :1015 deps:[JobId(1005)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.30259517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_read","method":"POST","req_id":"ec931f11-1ded-46d0-aac4-93c2e7238824","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":2501,"response_code":"200"}
Sep 22 23:21:42.303 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:21:42.304 DEBG Read      :1016 deps:[JobId(1006)] res:true
Sep 22 23:21:42.304 DEBG Read      :1016 deps:[JobId(1006)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.305922305Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_read","method":"POST","req_id":"59404c39-c66b-4aca-ac2f-8cc024f06818","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":2767,"response_code":"200"}
Sep 22 23:21:42.307 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:21:42.307 DEBG Read      :1017 deps:[JobId(1007)] res:true
Sep 22 23:21:42.308 DEBG Read      :1017 deps:[JobId(1007)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.310132596Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_read","method":"POST","req_id":"235a2621-7dfa-45e0-90d0-784023f1bc3d","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":3491,"response_code":"200"}
Sep 22 23:21:42.311 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:21:42.312 DEBG Read      :1018 deps:[JobId(1008)] res:true
Sep 22 23:21:42.312 DEBG Read      :1018 deps:[JobId(1008)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.314241424Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_read","method":"POST","req_id":"ea1f7246-1193-47ff-a583-f7c243e0d77c","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":3402,"response_code":"200"}
Sep 22 23:21:42.315 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:21:42.316 DEBG Read      :1019 deps:[JobId(1009)] res:true
Sep 22 23:21:42.316 DEBG Read      :1019 deps:[JobId(1009)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.318342751Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/bulk_read","method":"POST","req_id":"ebf2169c-368b-4f26-8229-7f436cd283d7","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":3393,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.319723358Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e/validate","method":"POST","req_id":"b0574342-119a-4c07-b4de-7f6e5e133ac4","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":303,"response_code":"200"}
Sep 22 23:21:42.320 DEBG Read      :1020 deps:[JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:21:42.321 DEBG Read      :1020 deps:[JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
Sep 22 23:21:42.321 DEBG Read      :1020 deps:[JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.328474963Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/f80a3498-aae7-4511-9946-9eb82ea06085/is_finished","method":"GET","req_id":"c0737c4e-3149-40c4-9c24-cbb8ecc6a7ca","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":205,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.329305037Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/f80a3498-aae7-4511-9946-9eb82ea06085/ok","method":"GET","req_id":"a673f4df-195f-4309-b0f7-b9a088ff721a","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":216,"response_code":"200"}
{"msg":"detach removing entry for volume 52ed3973-f354-40ba-a6f9-b5522a98667e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.33004055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume 52ed3973-f354-40ba-a6f9-b5522a98667e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.330072638Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:42.332 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:42.332 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
Sep 22 23:21:42.332 DEBG Flush     :1021 extent_limit None deps:[JobId(1020), JobId(1019), JobId(1018), JobId(1017), JobId(1016), JobId(1015), JobId(1014), JobId(1013), JobId(1012), JobId(1011), JobId(1010), JobId(1009), JobId(1008), JobId(1007), JobId(1006), JobId(1005), JobId(1004), JobId(1003), JobId(1002), JobId(1001), JobId(1000)] res:true f:1 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.333266496Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"81fb9c00-5d21-4891-b458-1cfd5eccef89 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.333297675Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.333541562Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/52ed3973-f354-40ba-a6f9-b5522a98667e","method":"DELETE","req_id":"755994e8-542f-40d5-871c-59cac93f93a8","remote_addr":"127.0.0.1:62417","local_addr":"127.0.0.1:59371","component":"dropshot","latency_us":3626,"response_code":"204"}
test test::test_pantry_validate_subset ... ok
Sep 22 23:21:42.509 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:42.524 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:42.539 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:42.545 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:21:42.545 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:21:42.545 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:21:42.635 DEBG IO Write 1002 has deps [JobId(1001)]
Sep 22 23:21:42.635 DEBG up_ds_listen was notified
Sep 22 23:21:42.635 DEBG up_ds_listen process 1002
Sep 22 23:21:42.635 DEBG [A] ack job 1002:3, : downstairs
Sep 22 23:21:42.635 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.70558976Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/e14704dd-f376-471e-a250-3ebae85bb495/is_finished","method":"GET","req_id":"0644f126-3797-4ef4-a330-ff93a2831140","remote_addr":"127.0.0.1:39199","local_addr":"127.0.0.1:47396","component":"dropshot","latency_us":258,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:42.786671513Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/7165c87e-0953-466c-822f-e26bef4d9d7d/is_finished","method":"GET","req_id":"0091e2b8-1f8c-4b7f-8c2d-78be4d76d22b","remote_addr":"127.0.0.1:57584","local_addr":"127.0.0.1:52393","component":"dropshot","latency_us":295,"response_code":"200"}
Sep 22 23:21:42.882 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.898 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.914 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:42.920 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:42.920 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:42.920 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:42.976 DEBG IO Write 1003 has deps [JobId(1001)]
Sep 22 23:21:42.976 DEBG up_ds_listen was notified
Sep 22 23:21:42.976 DEBG up_ds_listen process 1003
Sep 22 23:21:42.976 DEBG [A] ack job 1003:4, : downstairs
Sep 22 23:21:42.976 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:42.977 DEBG IO Flush 1004 has deps [JobId(1003), JobId(1002), JobId(1001)]
Sep 22 23:21:43.076 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:43.092 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:43.108 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:43.114 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:21:43.114 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:21:43.114 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:21:43.309 DEBG IO Write 1005 has deps [JobId(1004), JobId(1001)]
Sep 22 23:21:43.309 DEBG up_ds_listen was notified
Sep 22 23:21:43.309 DEBG up_ds_listen process 1005
Sep 22 23:21:43.309 DEBG [A] ack job 1005:6, : downstairs
Sep 22 23:21:43.309 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:43.503 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:43.515 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:43.527 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:43.531 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
Sep 22 23:21:43.531 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
Sep 22 23:21:43.531 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
Sep 22 23:21:43.645 DEBG IO Write 1006 has deps [JobId(1004), JobId(1001)]
Sep 22 23:21:43.645 DEBG up_ds_listen was notified
Sep 22 23:21:43.645 DEBG up_ds_listen process 1006
Sep 22 23:21:43.645 DEBG [A] ack job 1006:7, : downstairs
Sep 22 23:21:43.645 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:43.646 DEBG IO Flush 1007 has deps [JobId(1006), JobId(1005), JobId(1004)]
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:43.708309394Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/e14704dd-f376-471e-a250-3ebae85bb495/is_finished","method":"GET","req_id":"8a20dd5c-415f-4a0e-b6b1-63bf6f59b6d0","remote_addr":"127.0.0.1:39199","local_addr":"127.0.0.1:47396","component":"dropshot","latency_us":320,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:43.709247988Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/e14704dd-f376-471e-a250-3ebae85bb495/ok","method":"GET","req_id":"889ac11e-411e-4ef3-b3c7-da2104b0eea7","remote_addr":"127.0.0.1:39199","local_addr":"127.0.0.1:47396","component":"dropshot","latency_us":228,"response_code":"200"}
{"msg":"detach removing entry for volume 1eb9cf2b-fac5-48fd-86a9-e21629057f3e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:43.710022949Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume 1eb9cf2b-fac5-48fd-86a9-e21629057f3e","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:43.710054443Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:43.710 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
Sep 22 23:21:43.710 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
Sep 22 23:21:43.711 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:43.711669389Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:43.711699898Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:43.71195881Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/1eb9cf2b-fac5-48fd-86a9-e21629057f3e","method":"DELETE","req_id":"507c63fd-3dfc-47e3-8e95-048158e5a7ac","remote_addr":"127.0.0.1:39199","local_addr":"127.0.0.1:47396","component":"dropshot","latency_us":2072,"response_code":"204"}
Sep 22 23:21:43.807 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:43.822 DEBG Write     :1002 deps:[] res:true
Sep 22 23:21:43.839 DEBG Write     :1002 deps:[] res:true
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:43.841837632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/7165c87e-0953-466c-822f-e26bef4d9d7d/is_finished","method":"GET","req_id":"8b074832-3581-4beb-826d-716f6137fc76","remote_addr":"127.0.0.1:57584","local_addr":"127.0.0.1:52393","component":"dropshot","latency_us":408,"response_code":"200"}
Sep 22 23:21:43.846 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:21:43.846 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:21:43.846 DEBG Flush     :1003 extent_limit None deps:[JobId(1002)] res:true f:2 g:1
Sep 22 23:21:43.986 DEBG IO Write 1008 has deps [JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:43.986 DEBG up_ds_listen was notified
Sep 22 23:21:43.986 DEBG up_ds_listen process 1008
Sep 22 23:21:43.986 DEBG [A] ack job 1008:9, : downstairs
Sep 22 23:21:43.986 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:44.064 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:44.064 INFO Created new region file "/tmp/downstairs-RjobStuf/region.json"
Sep 22 23:21:44.164 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:44.164 INFO Opened existing region file "/tmp/downstairs-RjobStuf/region.json"
Sep 22 23:21:44.164 INFO Database read version 1
Sep 22 23:21:44.164 INFO Database write version 1
Sep 22 23:21:44.217 INFO UUID: 0c4b6300-aa56-4191-90c9-1d56da8be939
Sep 22 23:21:44.217 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:44.217 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:44.217 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:44.217 INFO Using address: 127.0.0.1:63038, task: main
Sep 22 23:21:44.218 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:44.218 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:53331, task: repair
Sep 22 23:21:44.218 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:53331, task: repair
Sep 22 23:21:44.219 INFO listening, local_addr: 127.0.0.1:53331, task: repair
Sep 22 23:21:44.219 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:53331, task: repair
Sep 22 23:21:44.219 INFO Using repair address: 127.0.0.1:53331, task: main
Sep 22 23:21:44.219 INFO No SSL acceptor configured, task: main
Sep 22 23:21:44.227 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:44.227 INFO Created new region file "/tmp/downstairs-CwBIEkKq/region.json"
Sep 22 23:21:44.322 DEBG IO Write 1009 has deps [JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:44.322 DEBG up_ds_listen was notified
Sep 22 23:21:44.322 DEBG up_ds_listen process 1009
Sep 22 23:21:44.322 DEBG [A] ack job 1009:10, : downstairs
Sep 22 23:21:44.322 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:44.323 DEBG IO Flush 1010 has deps [JobId(1009), JobId(1008), JobId(1007)]
Sep 22 23:21:44.330 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:44.331 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:44.331 INFO Opened existing region file "/tmp/downstairs-CwBIEkKq/region.json"
Sep 22 23:21:44.331 INFO Database read version 1
Sep 22 23:21:44.331 INFO Database write version 1
Sep 22 23:21:44.381 INFO UUID: 77625cc5-cf57-4c89-b7d2-c29dca5216c1
Sep 22 23:21:44.381 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:44.381 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:44.381 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:44.381 INFO Using address: 127.0.0.1:50898, task: main
Sep 22 23:21:44.381 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:44.382 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:43544, task: repair
Sep 22 23:21:44.382 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:43544, task: repair
Sep 22 23:21:44.382 INFO listening, local_addr: 127.0.0.1:43544, task: repair
Sep 22 23:21:44.382 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:43544, task: repair
Sep 22 23:21:44.382 INFO Using repair address: 127.0.0.1:43544, task: main
Sep 22 23:21:44.382 INFO No SSL acceptor configured, task: main
Sep 22 23:21:44.390 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:44.390 INFO Created new region file "/tmp/downstairs-A2wB4MTZ/region.json"
Sep 22 23:21:44.494 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:44.495 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:21:44.495 INFO Opened existing region file "/tmp/downstairs-A2wB4MTZ/region.json"
Sep 22 23:21:44.495 INFO Database read version 1
Sep 22 23:21:44.495 INFO Database write version 1
Sep 22 23:21:44.499 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:44.514 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:44.530 DEBG Write     :1004 deps:[] res:true
Sep 22 23:21:44.536 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:21:44.536 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:21:44.536 DEBG Flush     :1005 extent_limit None deps:[JobId(1004)] res:true f:3 g:1
Sep 22 23:21:44.546 INFO UUID: d89769b3-2748-4a20-ad6b-6b02a9292676
Sep 22 23:21:44.546 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:21:44.546 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:21:44.546 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:21:44.546 INFO Using address: 127.0.0.1:62519, task: main
Sep 22 23:21:44.546 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:21:44.546 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:46997, task: repair
Sep 22 23:21:44.546 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:46997, task: repair
Sep 22 23:21:44.546 INFO listening, local_addr: 127.0.0.1:46997, task: repair
Sep 22 23:21:44.546 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:46997, task: repair
Sep 22 23:21:44.546 INFO Using repair address: 127.0.0.1:46997, task: main
Sep 22 23:21:44.546 INFO No SSL acceptor configured, task: main
Sep 22 23:21:44.555 INFO Upstairs starts
Sep 22 23:21:44.555 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:44.555 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:44.555 INFO Crucible stats registered with UUID: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b
Sep 22 23:21:44.555 INFO Crucible 71f26a6c-b7c6-4f58-901b-ae4e36b1710b has session id: ca0bfa41-5904-420a-8eaa-b31e12bc4c5a
Sep 22 23:21:44.555 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:21:44.555 INFO [0] connecting to 127.0.0.1:63038, looper: 0
Sep 22 23:21:44.555 INFO [1] connecting to 127.0.0.1:50898, looper: 1
Sep 22 23:21:44.555 INFO [2] connecting to 127.0.0.1:62519, looper: 2
Sep 22 23:21:44.555 INFO up_listen starts, task: up_listen
Sep 22 23:21:44.555 INFO Wait for all three downstairs to come online
Sep 22 23:21:44.555 INFO Flush timeout: 0.5
Sep 22 23:21:44.556 INFO [0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected, looper: 0
Sep 22 23:21:44.556 INFO [0] Proc runs for 127.0.0.1:63038 in state New
Sep 22 23:21:44.556 INFO [1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected, looper: 1
Sep 22 23:21:44.556 INFO [1] Proc runs for 127.0.0.1:50898 in state New
Sep 22 23:21:44.556 INFO [2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected, looper: 2
Sep 22 23:21:44.556 INFO [2] Proc runs for 127.0.0.1:62519 in state New
Sep 22 23:21:44.556 INFO accepted connection from 127.0.0.1:40139, task: main
Sep 22 23:21:44.556 INFO accepted connection from 127.0.0.1:42516, task: main
Sep 22 23:21:44.556 INFO accepted connection from 127.0.0.1:55292, task: main
Sep 22 23:21:44.619 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:44.619 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:44.619 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:44.619 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:44.620 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:44.620 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:44.620 INFO [0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) New New New ds_transition to WaitActive
Sep 22 23:21:44.620 INFO [0] Transition from New to WaitActive
Sep 22 23:21:44.620 INFO [1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:44.620 INFO [1] Transition from New to WaitActive
Sep 22 23:21:44.620 INFO [2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:44.620 INFO [2] Transition from New to WaitActive
Sep 22 23:21:44.663 DEBG IO Write 1011 has deps [JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:44.664 DEBG up_ds_listen was notified
Sep 22 23:21:44.664 DEBG up_ds_listen process 1011
Sep 22 23:21:44.664 DEBG [A] ack job 1011:12, : downstairs
Sep 22 23:21:44.664 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:44.914 DEBG Write     :1006 deps:[] res:true
The guest has requested activation
Sep 22 23:21:44.917 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b active request set
Sep 22 23:21:44.917 INFO [0] received activate with gen 1
Sep 22 23:21:44.917 INFO [0] client got ds_active_rx, promote! session 14046fa9-5850-443d-9708-a7d5d0130e56
Sep 22 23:21:44.917 INFO [1] received activate with gen 1
Sep 22 23:21:44.917 INFO [1] client got ds_active_rx, promote! session 14046fa9-5850-443d-9708-a7d5d0130e56
Sep 22 23:21:44.918 INFO [2] received activate with gen 1
Sep 22 23:21:44.918 INFO [2] client got ds_active_rx, promote! session 14046fa9-5850-443d-9708-a7d5d0130e56
Sep 22 23:21:44.918 INFO UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } is now active (read-write)
Sep 22 23:21:44.918 INFO UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } is now active (read-write)
Sep 22 23:21:44.918 INFO UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } is now active (read-write)
Sep 22 23:21:44.919 INFO [0] downstairs client at 127.0.0.1:63038 has UUID 0c4b6300-aa56-4191-90c9-1d56da8be939
Sep 22 23:21:44.919 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 0c4b6300-aa56-4191-90c9-1d56da8be939, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:44.919 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitActive WaitActive WaitActive
Sep 22 23:21:44.919 INFO [1] downstairs client at 127.0.0.1:50898 has UUID 77625cc5-cf57-4c89-b7d2-c29dca5216c1
Sep 22 23:21:44.919 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 77625cc5-cf57-4c89-b7d2-c29dca5216c1, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:44.919 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitActive WaitActive WaitActive
Sep 22 23:21:44.919 INFO [2] downstairs client at 127.0.0.1:62519 has UUID d89769b3-2748-4a20-ad6b-6b02a9292676
Sep 22 23:21:44.919 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: d89769b3-2748-4a20-ad6b-6b02a9292676, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:21:44.919 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitActive WaitActive WaitActive
Sep 22 23:21:44.927 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:44.931 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:44.932 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:44.934 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:44.939 DEBG Write     :1006 deps:[] res:true
Sep 22 23:21:44.943 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
Sep 22 23:21:44.943 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
Sep 22 23:21:44.943 DEBG Flush     :1007 extent_limit None deps:[JobId(1006)] res:true f:4 g:1
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:44.947077283Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/7165c87e-0953-466c-822f-e26bef4d9d7d/is_finished","method":"GET","req_id":"ec4c93f1-d209-413c-87f1-28c3dc8512e4","remote_addr":"127.0.0.1:57584","local_addr":"127.0.0.1:52393","component":"dropshot","latency_us":224,"response_code":"200"}
{"msg":"job 7165c87e-0953-466c-822f-e26bef4d9d7d failed with sha256 digest mismatch! expected 00000000000000000000000000000000000000000000000000000000f5b32221, saw 319d678f093c43502ca360911d52b475dea7fa6dcd962150c84fff18f5b32221","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:44.947729187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:44.947781689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/7165c87e-0953-466c-822f-e26bef4d9d7d/ok","method":"GET","req_id":"e1071ba7-2d11-4082-8795-0fd6a9b8d245","remote_addr":"127.0.0.1:57584","local_addr":"127.0.0.1:52393","component":"dropshot","latency_us":210,"response_code":"200"}
{"msg":"detach removing entry for volume 3251378f-ea13-40c9-9d37-7f38d07fbf35","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:44.948384269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume 3251378f-ea13-40c9-9d37-7f38d07fbf35","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:44.948434552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:44.948 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
Sep 22 23:21:44.948 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
Sep 22 23:21:44.948 DEBG Flush     :1008 extent_limit None deps:[] res:true f:5 g:1
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:44.949412959Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"cf566b9e-aff0-4665-9031-98e179d159a4 set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:44.9494643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:44.949614502Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/3251378f-ea13-40c9-9d37-7f38d07fbf35","method":"DELETE","req_id":"4837a099-cd81-4a19-9262-885c3151a9f6","remote_addr":"127.0.0.1:57584","local_addr":"127.0.0.1:52393","component":"dropshot","latency_us":1337,"response_code":"204"}
Sep 22 23:21:44.951 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:44.952 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:44.953 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:44.953 INFO [0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:44.953 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:44.953 WARN [0] new RM replaced this: None
Sep 22 23:21:44.953 INFO [0] Starts reconcile loop
Sep 22 23:21:44.954 INFO [1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:44.954 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:44.954 WARN [1] new RM replaced this: None
Sep 22 23:21:44.954 INFO [1] Starts reconcile loop
Sep 22 23:21:44.954 INFO [2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:44.954 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:44.954 WARN [2] new RM replaced this: None
Sep 22 23:21:44.954 INFO [2] Starts reconcile loop
Sep 22 23:21:44.954 INFO [0] 127.0.0.1:63038 task reports connection:true
Sep 22 23:21:44.954 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:44.954 INFO [0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:44.954 INFO [0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:44.954 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:44.954 INFO [1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:44.954 INFO [1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:44.954 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:44.954 INFO [2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:44.954 INFO [2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:44.954 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:44.954 INFO Max found gen is 1
Sep 22 23:21:44.954 INFO Generation requested: 1 >= found:1
Sep 22 23:21:44.954 INFO Next flush: 1
Sep 22 23:21:44.954 INFO All extents match
Sep 22 23:21:44.954 INFO No downstairs repair required
Sep 22 23:21:44.954 INFO No initial repair work was required
Sep 22 23:21:44.954 INFO Set Downstairs and Upstairs active
Sep 22 23:21:44.954 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b is now active with session: 14046fa9-5850-443d-9708-a7d5d0130e56
Sep 22 23:21:44.954 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Set Active after no repair
Sep 22 23:21:44.954 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:44.954 INFO Set check for repair
Sep 22 23:21:44.954 INFO [1] 127.0.0.1:50898 task reports connection:true
Sep 22 23:21:44.954 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Active Active Active
Sep 22 23:21:44.954 INFO Set check for repair
Sep 22 23:21:44.954 INFO [2] 127.0.0.1:62519 task reports connection:true
Sep 22 23:21:44.954 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Active Active Active
Sep 22 23:21:44.954 INFO Set check for repair
Sep 22 23:21:44.954 INFO [0] received reconcile message
Sep 22 23:21:44.954 INFO [0] All repairs completed, exit
Sep 22 23:21:44.954 INFO [0] Starts cmd_loop
Sep 22 23:21:44.955 INFO [1] received reconcile message
Sep 22 23:21:44.955 INFO [1] All repairs completed, exit
Sep 22 23:21:44.955 INFO [1] Starts cmd_loop
Sep 22 23:21:44.955 INFO [2] received reconcile message
Sep 22 23:21:44.955 INFO [2] All repairs completed, exit
Sep 22 23:21:44.955 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:44.976 DEBG IO Read  1000 has deps []
Sep 22 23:21:44.999 DEBG IO Write 1012 has deps [JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:45.000 DEBG up_ds_listen was notified
Sep 22 23:21:45.000 DEBG up_ds_listen process 1012
Sep 22 23:21:45.000 DEBG [A] ack job 1012:13, : downstairs
Sep 22 23:21:45.000 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:45.000 DEBG IO Flush 1013 has deps [JobId(1012), JobId(1011), JobId(1010)]
Sep 22 23:21:45.007 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:45.036 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:45.042 INFO Upstairs starts
Sep 22 23:21:45.042 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:21:45.042 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:21:45.042 INFO Crucible stats registered with UUID: 40e06af7-e69f-43e2-974b-bb00dc5e8960
Sep 22 23:21:45.042 INFO Crucible 40e06af7-e69f-43e2-974b-bb00dc5e8960 has session id: 82d5f8f4-faa4-453f-a84d-0d23406d6cd8
Sep 22 23:21:45.043 INFO [0] connecting to 127.0.0.1:55664, looper: 0
Sep 22 23:21:45.043 INFO [1] connecting to 127.0.0.1:37034, looper: 1
Sep 22 23:21:45.043 INFO [2] connecting to 127.0.0.1:53040, looper: 2
Sep 22 23:21:45.043 INFO up_listen starts, task: up_listen
Sep 22 23:21:45.043 INFO Wait for all three downstairs to come online
Sep 22 23:21:45.043 INFO Flush timeout: 0.5
Sep 22 23:21:45.043 INFO [0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 looper connected, looper: 0
Sep 22 23:21:45.043 INFO [0] Proc runs for 127.0.0.1:55664 in state New
Sep 22 23:21:45.044 INFO [1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 looper connected, looper: 1
Sep 22 23:21:45.044 INFO [1] Proc runs for 127.0.0.1:37034 in state New
Sep 22 23:21:45.044 INFO [2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 looper connected, looper: 2
Sep 22 23:21:45.044 INFO [2] Proc runs for 127.0.0.1:53040 in state New
Sep 22 23:21:45.044 INFO accepted connection from 127.0.0.1:47023, task: main
Sep 22 23:21:45.044 INFO accepted connection from 127.0.0.1:39586, task: main
Sep 22 23:21:45.044 INFO accepted connection from 127.0.0.1:54462, task: main
Sep 22 23:21:45.045 INFO Connection request from 40e06af7-e69f-43e2-974b-bb00dc5e8960 with version 4, task: proc
Sep 22 23:21:45.045 INFO upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:45.045 INFO Connection request from 40e06af7-e69f-43e2-974b-bb00dc5e8960 with version 4, task: proc
Sep 22 23:21:45.045 INFO upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:45.045 INFO Connection request from 40e06af7-e69f-43e2-974b-bb00dc5e8960 with version 4, task: proc
Sep 22 23:21:45.045 INFO upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:45.045 INFO [0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (f7ed2bae-5c71-405b-9355-9393f8d7847d) New New New ds_transition to WaitActive
Sep 22 23:21:45.045 INFO [0] Transition from New to WaitActive
Sep 22 23:21:45.045 INFO [1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (f7ed2bae-5c71-405b-9355-9393f8d7847d) WaitActive New New ds_transition to WaitActive
Sep 22 23:21:45.045 INFO [1] Transition from New to WaitActive
Sep 22 23:21:45.045 INFO [2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (f7ed2bae-5c71-405b-9355-9393f8d7847d) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:21:45.045 INFO [2] Transition from New to WaitActive
The guest has requested activation
Sep 22 23:21:45.046 INFO 40e06af7-e69f-43e2-974b-bb00dc5e8960 active request set
Sep 22 23:21:45.046 INFO [0] received activate with gen 2
Sep 22 23:21:45.046 INFO [0] client got ds_active_rx, promote! session f7ed2bae-5c71-405b-9355-9393f8d7847d
Sep 22 23:21:45.046 INFO [1] received activate with gen 2
Sep 22 23:21:45.046 INFO [1] client got ds_active_rx, promote! session f7ed2bae-5c71-405b-9355-9393f8d7847d
Sep 22 23:21:45.046 INFO [2] received activate with gen 2
Sep 22 23:21:45.046 INFO [2] client got ds_active_rx, promote! session f7ed2bae-5c71-405b-9355-9393f8d7847d
Sep 22 23:21:45.046 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } to UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 }
Sep 22 23:21:45.046 WARN Signaling to UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } thread that UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } is being promoted (read-write)
Sep 22 23:21:45.046 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } to UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 }
Sep 22 23:21:45.046 WARN Signaling to UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } thread that UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } is being promoted (read-write)
Sep 22 23:21:45.047 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } to UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 }
Sep 22 23:21:45.047 WARN Signaling to UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } thread that UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } is being promoted (read-write)
Sep 22 23:21:45.047 WARN Another upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 }, task: main
Sep 22 23:21:45.047 INFO UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } is now active (read-write)
Sep 22 23:21:45.047 WARN Another upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 }, task: main
Sep 22 23:21:45.047 INFO UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } is now active (read-write)
Sep 22 23:21:45.047 WARN Another upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 }, task: main
Sep 22 23:21:45.048 INFO UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: f7ed2bae-5c71-405b-9355-9393f8d7847d, gen: 2 } is now active (read-write)
Sep 22 23:21:45.048 INFO connection (127.0.0.1:49946): all done
Sep 22 23:21:45.048 INFO connection (127.0.0.1:45897): all done
Sep 22 23:21:45.048 INFO connection (127.0.0.1:36645): all done
{"msg":"[0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) cmd_loop saw YouAreNoLongerActive 40e06af7-e69f-43e2-974b-bb00dc5e8960 f7ed2bae-5c71-405b-9355-9393f8d7847d 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:45.048999327Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) Active Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049072867Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049112226Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 set inactive, session d90ddc5f-b441-432b-bb73-5ab29b73ba5b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049162794Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:55664: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:45.049205203Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049246664Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 connection to 127.0.0.1:55664 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049289571Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) cmd_loop saw YouAreNoLongerActive 40e06af7-e69f-43e2-974b-bb00dc5e8960 f7ed2bae-5c71-405b-9355-9393f8d7847d 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:45.049342474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) Disconnected Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049387072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049432073Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 set inactive, session d90ddc5f-b441-432b-bb73-5ab29b73ba5b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.0494717Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:37034: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:45.049510855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049559363Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 connection to 127.0.0.1:37034 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049600997Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) cmd_loop saw YouAreNoLongerActive 40e06af7-e69f-43e2-974b-bb00dc5e8960 f7ed2bae-5c71-405b-9355-9393f8d7847d 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:45.049646415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) Disconnected Disconnected Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049687473Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049723056Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 set inactive, session d90ddc5f-b441-432b-bb73-5ab29b73ba5b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049761084Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:45.049 INFO [0] downstairs client at 127.0.0.1:55664 has UUID b740b481-b810-4686-a6ad-5e7a129e9669
{"msg":"127.0.0.1:53040: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:45.049799847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
Sep 22 23:21:45.049 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: b740b481-b810-4686-a6ad-5e7a129e9669, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049850266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:45.049 INFO 40e06af7-e69f-43e2-974b-bb00dc5e8960 WaitActive WaitActive WaitActive
{"msg":"[2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 connection to 127.0.0.1:53040 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049890626Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:45.049929026Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:45.049 INFO [1] downstairs client at 127.0.0.1:37034 has UUID efa5c554-210a-4196-815d-fa3d1dfbb858
Sep 22 23:21:45.049 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: efa5c554-210a-4196-815d-fa3d1dfbb858, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[0] 127.0.0.1:55664 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.049970165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:45.050 INFO 40e06af7-e69f-43e2-974b-bb00dc5e8960 WaitActive WaitActive WaitActive
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.050008607Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:55664 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.050044531Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:45.050 INFO [2] downstairs client at 127.0.0.1:53040 has UUID e1ba5200-2113-47b5-8eab-dc553c509bb6
Sep 22 23:21:45.050 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: e1ba5200-2113-47b5-8eab-dc553c509bb6, encrypted: true, database_read_version: 1, database_write_version: 1 }
{"msg":"[1] 127.0.0.1:37034 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.050087873Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:45.050 INFO 40e06af7-e69f-43e2-974b-bb00dc5e8960 WaitActive WaitActive WaitActive
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.050129028Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:37034 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.050167109Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:53040 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.050203482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"40e06af7-e69f-43e2-974b-bb00dc5e8960 Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.050240689Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:53040 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:45.050276118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:45.050320475Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:45.050357229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
test test::test_pantry_import_from_url_ovmf_bad_digest ... ok
Sep 22 23:21:45.064 INFO Current flush_numbers [0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:21:45.066 INFO Current flush_numbers [0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:21:45.067 INFO Current flush_numbers [0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:21:45.072 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:45.087 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:45.089 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:45.090 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:45.091 INFO [0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (f7ed2bae-5c71-405b-9355-9393f8d7847d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:45.091 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:21:45.091 WARN [0] new RM replaced this: None
Sep 22 23:21:45.091 INFO [0] Starts reconcile loop
Sep 22 23:21:45.091 INFO [1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (f7ed2bae-5c71-405b-9355-9393f8d7847d) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:21:45.091 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:21:45.091 WARN [1] new RM replaced this: None
Sep 22 23:21:45.091 INFO [1] Starts reconcile loop
Sep 22 23:21:45.091 INFO [2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (f7ed2bae-5c71-405b-9355-9393f8d7847d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:21:45.091 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:21:45.091 WARN [2] new RM replaced this: None
Sep 22 23:21:45.092 INFO [2] Starts reconcile loop
Sep 22 23:21:45.092 INFO [0] 127.0.0.1:55664 task reports connection:true
Sep 22 23:21:45.092 INFO 40e06af7-e69f-43e2-974b-bb00dc5e8960 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:21:45.092 INFO [0]R flush_numbers[0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:21:45.092 INFO [0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]
Sep 22 23:21:45.092 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:45.092 INFO [1]R flush_numbers[0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:21:45.092 INFO [1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]
Sep 22 23:21:45.092 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:45.092 INFO [2]R flush_numbers[0..12]: [1, 1, 2, 2, 3, 3, 4, 4, 0, 0, 0, 0]
Sep 22 23:21:45.092 INFO [2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]
Sep 22 23:21:45.092 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:21:45.092 INFO Max found gen is 2
Sep 22 23:21:45.092 INFO Generation requested: 2 >= found:2
Sep 22 23:21:45.092 INFO Next flush: 5
Sep 22 23:21:45.092 INFO All extents match
Sep 22 23:21:45.092 INFO No downstairs repair required
Sep 22 23:21:45.092 INFO No initial repair work was required
Sep 22 23:21:45.092 INFO Set Downstairs and Upstairs active
Sep 22 23:21:45.092 INFO 40e06af7-e69f-43e2-974b-bb00dc5e8960 is now active with session: f7ed2bae-5c71-405b-9355-9393f8d7847d
Sep 22 23:21:45.092 INFO 40e06af7-e69f-43e2-974b-bb00dc5e8960 Set Active after no repair
Sep 22 23:21:45.092 INFO Notify all downstairs, region set compare is done.
Sep 22 23:21:45.092 INFO Set check for repair
Sep 22 23:21:45.092 INFO [1] 127.0.0.1:37034 task reports connection:true
Sep 22 23:21:45.092 INFO 40e06af7-e69f-43e2-974b-bb00dc5e8960 Active Active Active
Sep 22 23:21:45.092 INFO Set check for repair
Sep 22 23:21:45.092 INFO [2] 127.0.0.1:53040 task reports connection:true
Sep 22 23:21:45.092 INFO 40e06af7-e69f-43e2-974b-bb00dc5e8960 Active Active Active
Sep 22 23:21:45.092 INFO Set check for repair
Sep 22 23:21:45.092 INFO [0] received reconcile message
Sep 22 23:21:45.093 INFO [0] All repairs completed, exit
Sep 22 23:21:45.093 INFO [0] Starts cmd_loop
Sep 22 23:21:45.093 INFO [1] received reconcile message
Sep 22 23:21:45.093 INFO [1] All repairs completed, exit
Sep 22 23:21:45.093 INFO [1] Starts cmd_loop
Sep 22 23:21:45.093 INFO [2] received reconcile message
Sep 22 23:21:45.093 INFO [2] All repairs completed, exit
Sep 22 23:21:45.093 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:21:45.098 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:21:45.102 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:45.103 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:45.104 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:45.105 DEBG IO Read  1000 has deps []
Sep 22 23:21:45.139 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:45.202 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:45.269 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:45.325 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:45.363 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:45.401 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:45.402 DEBG up_ds_listen was notified
Sep 22 23:21:45.403 DEBG up_ds_listen process 1000
Sep 22 23:21:45.403 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:45.404 DEBG IO Write 1014 has deps [JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:45.404 DEBG up_ds_listen was notified
Sep 22 23:21:45.404 DEBG up_ds_listen process 1014
Sep 22 23:21:45.404 DEBG [A] ack job 1014:15, : downstairs
Sep 22 23:21:45.404 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:45.501 DEBG up_ds_listen process 1001
Sep 22 23:21:45.501 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:45.501 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:21:45.501 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:21:45.501 DEBG up_ds_listen was notified
Sep 22 23:21:45.501 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:21:45.550 WARN returning error on write!
Sep 22 23:21:45.550 DEBG Write     :1000 deps:[] res:false
Sep 22 23:21:45.551 WARN returning error on write!
Sep 22 23:21:45.551 DEBG Write     :1000 deps:[] res:false
Sep 22 23:21:45.581 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:45.612 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:45.613 INFO [lossy] skipping 1001
Sep 22 23:21:45.613 INFO [lossy] skipping 1001
Sep 22 23:21:45.614 INFO [lossy] skipping 1000
Sep 22 23:21:45.615 WARN returning error on write!
Sep 22 23:21:45.615 DEBG Write     :1000 deps:[] res:false
Sep 22 23:21:45.615 INFO [lossy] skipping 1000
Sep 22 23:21:45.616 WARN returning error on write!
Sep 22 23:21:45.616 DEBG Write     :1000 deps:[] res:false
Sep 22 23:21:45.646 DEBG Write     :1000 deps:[] res:true
Sep 22 23:21:45.656 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:45.656 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:45.657 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:21:45.956 INFO Checking if live repair is needed
Sep 22 23:21:45.956 INFO No Live Repair required at this time
Sep 22 23:21:45.989 DEBG IO Write 1015 has deps [JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:45.989 DEBG IO Flush 1016 has deps [JobId(1015), JobId(1014), JobId(1013)]
Sep 22 23:21:45.989 DEBG up_ds_listen was notified
Sep 22 23:21:45.989 DEBG up_ds_listen process 1015
Sep 22 23:21:45.989 DEBG [A] ack job 1015:16, : downstairs
Sep 22 23:21:45.989 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:46.229 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:21:46.319 DEBG IO Write 1017 has deps [JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:46.319 DEBG up_ds_listen was notified
Sep 22 23:21:46.320 DEBG up_ds_listen process 1017
Sep 22 23:21:46.320 DEBG [A] ack job 1017:18, : downstairs
Sep 22 23:21:46.320 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:46.655 DEBG IO Write 1018 has deps [JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:46.655 DEBG up_ds_listen was notified
Sep 22 23:21:46.655 DEBG up_ds_listen process 1018
Sep 22 23:21:46.655 DEBG [A] ack job 1018:19, : downstairs
Sep 22 23:21:46.655 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:46.656 DEBG IO Flush 1019 has deps [JobId(1018), JobId(1017), JobId(1016)]
Sep 22 23:21:46.857 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:21:46.983 INFO Request to deactivate this guest
Sep 22 23:21:46.983 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b set deactivating.
Sep 22 23:21:46.983 DEBG No work, no need to flush, return OK
note: configured to log to "/dev/stdout"
{"msg":"listening","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:46.984867303Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:49824","component":"dropshot"}
{"msg":"listen IP: 127.0.0.1:49824","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:46.985083066Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:46.991 DEBG IO Write 1020 has deps [JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:46.992 DEBG up_ds_listen was notified
Sep 22 23:21:46.992 DEBG up_ds_listen process 1020
Sep 22 23:21:46.992 DEBG [A] ack job 1020:21, : downstairs
Sep 22 23:21:46.992 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"accepted connection","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.054382953Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"local_addr":"127.0.0.1:49824","component":"dropshot","remote_addr":"127.0.0.1:33149"}
{"msg":"no entry exists for volume 3dd77b9b-1720-4c07-b5d9-e4b24cceb521, constructing...","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055097208Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055330642Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055409699Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055458003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible stats registered with UUID: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055502869Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Crucible 71f26a6c-b7c6-4f58-901b-ae4e36b1710b has session id: 9be27365-d3f4-4c51-a131-3bf3779747df","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055551651Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] connecting to 127.0.0.1:63038","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055596206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] connecting to 127.0.0.1:50898","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.05566658Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] connecting to 127.0.0.1:62519","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055799078Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055852954Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile","task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055900614Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.055954091Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.119690859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:63038 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.119750173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.119790941Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:50898 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.119828909Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.119864746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
Sep 22 23:21:47.119 INFO accepted connection from 127.0.0.1:50361, task: main
{"msg":"[2] Proc runs for 127.0.0.1:62519 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.11990233Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:47.119 INFO accepted connection from 127.0.0.1:56375, task: main
Sep 22 23:21:47.120 INFO accepted connection from 127.0.0.1:64643, task: main
Sep 22 23:21:47.120 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:47.120 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:47.120 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:47.120 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } connected, version 4, task: proc
Sep 22 23:21:47.120 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:47.120 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } connected, version 4, task: proc
{"msg":"[0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.120673996Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.120722734Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.120763191Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.120798224Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.120837845Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.120880921Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:47.325 DEBG IO Write 1021 has deps [JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:47.325 DEBG up_ds_listen was notified
Sep 22 23:21:47.325 DEBG up_ds_listen process 1021
Sep 22 23:21:47.325 DEBG [A] ack job 1021:22, : downstairs
Sep 22 23:21:47.325 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:47.326 DEBG IO Flush 1022 has deps [JobId(1021), JobId(1020), JobId(1019)]
{"msg":"volume 3dd77b9b-1720-4c07-b5d9-e4b24cceb521 constructed ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.387557591Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has requested activation
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.38780196Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received activate with gen 2","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.387859134Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] client got ds_active_rx, promote! session 3d6550fa-8c01-4792-b110-6653df927fd3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.387905932Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received activate with gen 2","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.387949696Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] client got ds_active_rx, promote! session 3d6550fa-8c01-4792-b110-6653df927fd3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.387995725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received activate with gen 2","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.388040649Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:47.388 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 }
{"msg":"[2] client got ds_active_rx, promote! session 3d6550fa-8c01-4792-b110-6653df927fd3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.388147081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:47.388 WARN Signaling to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } thread that UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } is being promoted (read-write)
Sep 22 23:21:47.388 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 }
Sep 22 23:21:47.388 WARN Signaling to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } thread that UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } is being promoted (read-write)
Sep 22 23:21:47.388 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 }
Sep 22 23:21:47.388 WARN Signaling to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } thread that UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } is being promoted (read-write)
Sep 22 23:21:47.388 WARN Another upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 }, task: main
Sep 22 23:21:47.388 INFO UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } is now active (read-write)
Sep 22 23:21:47.388 WARN Another upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 }, task: main
Sep 22 23:21:47.389 INFO UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } is now active (read-write)
Sep 22 23:21:47.389 WARN Another upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 }, task: main
Sep 22 23:21:47.389 INFO UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } is now active (read-write)
Sep 22 23:21:47.389 INFO connection (127.0.0.1:40139): all done
Sep 22 23:21:47.389 INFO connection (127.0.0.1:42516): all done
Sep 22 23:21:47.389 INFO connection (127.0.0.1:55292): all done
Sep 22 23:21:47.389 ERRO [0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) cmd_loop saw YouAreNoLongerActive 71f26a6c-b7c6-4f58-901b-ae4e36b1710b 3d6550fa-8c01-4792-b110-6653df927fd3 2
Sep 22 23:21:47.389 INFO [0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) Active Active Active ds_transition to Disabled
Sep 22 23:21:47.389 INFO [0] Transition from Active to Disabled
Sep 22 23:21:47.389 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b set inactive, session 14046fa9-5850-443d-9708-a7d5d0130e56
Sep 22 23:21:47.389 ERRO 127.0.0.1:63038: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1, looper: 0
Sep 22 23:21:47.389 INFO [0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Gone missing, transition from Disabled to Disconnected
Sep 22 23:21:47.389 INFO [0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b connection to 127.0.0.1:63038 closed, looper: 0
Sep 22 23:21:47.389 ERRO [1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) cmd_loop saw YouAreNoLongerActive 71f26a6c-b7c6-4f58-901b-ae4e36b1710b 3d6550fa-8c01-4792-b110-6653df927fd3 2
Sep 22 23:21:47.389 INFO [1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) Disconnected Active Active ds_transition to Disabled
Sep 22 23:21:47.389 INFO [1] Transition from Active to Disabled
Sep 22 23:21:47.389 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b set inactive, session 14046fa9-5850-443d-9708-a7d5d0130e56
Sep 22 23:21:47.389 ERRO 127.0.0.1:50898: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1, looper: 1
Sep 22 23:21:47.390 INFO [1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Gone missing, transition from Disabled to Disconnected
Sep 22 23:21:47.390 INFO [1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b connection to 127.0.0.1:50898 closed, looper: 1
Sep 22 23:21:47.390 ERRO [2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) cmd_loop saw YouAreNoLongerActive 71f26a6c-b7c6-4f58-901b-ae4e36b1710b 3d6550fa-8c01-4792-b110-6653df927fd3 2
Sep 22 23:21:47.390 INFO [2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) Disconnected Disconnected Active ds_transition to Disabled
Sep 22 23:21:47.390 INFO [2] Transition from Active to Disabled
Sep 22 23:21:47.390 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b set inactive, session 14046fa9-5850-443d-9708-a7d5d0130e56
Sep 22 23:21:47.390 ERRO 127.0.0.1:62519: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1, looper: 2
Sep 22 23:21:47.390 INFO [2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Gone missing, transition from Disabled to Disconnected
Sep 22 23:21:47.390 INFO [2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b connection to 127.0.0.1:62519 closed, looper: 2
Sep 22 23:21:47.390 WARN [0] pm_task rx.recv() is None
Sep 22 23:21:47.390 INFO [0] 127.0.0.1:63038 task reports connection:false
Sep 22 23:21:47.390 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Disconnected Disconnected Disconnected
Sep 22 23:21:47.390 INFO [0] 127.0.0.1:63038 task reports offline
Sep 22 23:21:47.390 INFO [1] 127.0.0.1:50898 task reports connection:false
Sep 22 23:21:47.390 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Disconnected Disconnected Disconnected
Sep 22 23:21:47.390 INFO [1] 127.0.0.1:50898 task reports offline
Sep 22 23:21:47.390 INFO [2] 127.0.0.1:62519 task reports connection:false
Sep 22 23:21:47.390 INFO 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Disconnected Disconnected Disconnected
Sep 22 23:21:47.390 INFO [2] 127.0.0.1:62519 task reports offline
Sep 22 23:21:47.390 WARN [1] pm_task rx.recv() is None
Sep 22 23:21:47.390 WARN [2] pm_task rx.recv() is None
{"msg":"[0] downstairs client at 127.0.0.1:63038 has UUID 0c4b6300-aa56-4191-90c9-1d56da8be939","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.390911429Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 0c4b6300-aa56-4191-90c9-1d56da8be939, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.390975284Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.391025466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] downstairs client at 127.0.0.1:50898 has UUID 77625cc5-cf57-4c89-b7d2-c29dca5216c1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.391074256Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 77625cc5-cf57-4c89-b7d2-c29dca5216c1, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.391128209Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.391176309Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] downstairs client at 127.0.0.1:62519 has UUID d89769b3-2748-4a20-ad6b-6b02a9292676","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.391221944Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: d89769b3-2748-4a20-ad6b-6b02a9292676, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.39127498Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.391318245Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:47.398 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:47.399 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:47.400 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:21:47.413 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:47.414 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:47.415 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.415393585Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.415450422Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:47.415491788Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.415527659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.415735838Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.415789586Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:47.415829967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.415865905Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416072929Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416125906Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:47.416169212Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416206974Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:63038 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416252173Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416293939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416332311Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416368143Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416411659Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416456262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416501286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
The guest has finished waiting for activation
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416547677Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416592846Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416638011Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416685939Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Max found gen is 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416730102Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Generation requested: 2 >= found:1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416775419Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Next flush: 1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416819267Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416863024Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416903392Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416942014Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.416988712Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b is now active with session: 3d6550fa-8c01-4792-b110-6653df927fd3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417031104Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.41707108Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417115351Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417157393Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:50898 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417196286Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417237081Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417279346Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:62519 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.41732164Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.41736413Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417404387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417444288Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417482165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417517956Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417553694Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417590852Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417630437Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417671391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.41771967Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417763415Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 3dd77b9b-1720-4c07-b5d9-e4b24cceb521 activated ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.41780832Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"volume 3dd77b9b-1720-4c07-b5d9-e4b24cceb521 constructed and inserted ok","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.41785417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.417895039Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/3dd77b9b-1720-4c07-b5d9-e4b24cceb521","method":"POST","req_id":"37378f22-ea19-4ef3-8f88-c6c7f7d630a2","remote_addr":"127.0.0.1:33149","local_addr":"127.0.0.1:49824","component":"dropshot","latency_us":362314,"response_code":"200"}
{"msg":"Scrub check for 3dd77b9b-1720-4c07-b5d9-e4b24cceb521","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.418174772Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Scrub for 3dd77b9b-1720-4c07-b5d9-e4b24cceb521 begins","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.418244142Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Scrub with total_size:1966080 block_size:512","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.418292746Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Scrubs from block 0 to 3840 in (256) 131072 size IOs pm:0","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.418332612Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.418580861Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/3dd77b9b-1720-4c07-b5d9-e4b24cceb521/scrub","method":"POST","req_id":"b4b9054e-0bf0-4378-b5dd-d72b3ecef36b","remote_addr":"127.0.0.1:33149","local_addr":"127.0.0.1:49824","component":"dropshot","latency_us":638,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.419367918Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/9890f602-aa95-4bcc-8172-78f624d84944/is_finished","method":"GET","req_id":"0560e9e5-8850-4ece-87fd-66ef564218e8","remote_addr":"127.0.0.1:33149","local_addr":"127.0.0.1:49824","component":"dropshot","latency_us":184,"response_code":"200"}
Sep 22 23:21:47.479 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:21:47.484 DEBG up_ds_listen was notified
Sep 22 23:21:47.484 DEBG up_ds_listen process 1000
Sep 22 23:21:47.485 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:47.584 DEBG up_ds_listen checked 1 jobs, back to waiting
0 512 ok
512 1024 ok
1024 1536 ok
1536 2048 ok
2048 2560 ok
2560 3072 ok
3072 3584 ok
3584 4096 ok
4096 4608 ok
4608 5120 ok
5120 5632 ok
5632 6144 ok
6144 6656 ok
6656 7168 ok
7168 7680 ok
7680 8192 ok
8192 8704 ok
8704 9216 ok
9216 9728 ok
9728 10240 ok
10240 10752 ok
10752 11264 ok
11264 11776 ok
11776 12288 ok
12288 12800 ok
12800 13312 ok
13312 13824 ok
13824 14336 ok
14336 14848 ok
14848 15360 ok
15360 15872 ok
15872 16384 ok
16384 16896 ok
16896 17408 ok
17408 17920 ok
17920 18432 ok
18432 18944 ok
18944 19456 ok
19456 19968 ok
19968 20480 ok
20480 20992 ok
20992 21504 ok
21504 22016 ok
22016 22528 ok
22528 23040 ok
23040 23552 ok
23552 24064 ok
24064 24576 ok
24576 25088 ok
25088 25600 ok
25600 26112 ok
26112 26624 ok
26624 27136 ok
27136 27648 ok
27648 28160 ok
28160 28672 ok
28672 29184 ok
29184 29696 ok
29696 30208 ok
30208 30720 ok
30720 31232 ok
31232 31744 ok
31744 32256 ok
32256 32768 ok
32768 33280 ok
33280 33792 ok
33792 34304 ok
34304 34816 ok
34816 35328 ok
35328 35840 ok
35840 36352 ok
36352 36864 ok
36864 37376 ok
37376 37888 ok
37888 38400 ok
38400 38912 ok
38912 39424 ok
39424 39936 ok
39936 40448 ok
40448 40960 ok
40960 41472 ok
41472 41984 ok
41984 42496 ok
42496 43008 ok
43008 43520 ok
43520 44032 ok
44032 44544 ok
44544 45056 ok
45056 45568 ok
45568 46080 ok
46080 46592 ok
46592 47104 ok
47104 47616 ok
47616 48128 ok
48128 48640 ok
48640 49152 ok
49152 49664 ok
49664 50176 ok
50176 50688 ok
50688 51200 ok
51200 51712 ok
51712 52224 ok
52224 52736 ok
52736 53248 ok
53248 53760 ok
53760 54272 ok
54272 54784 ok
54784 55296 ok
55296 55808 ok
55808 56320 ok
56320 56832 ok
56832 57344 ok
57344 57856 ok
57856 58368 ok
58368 58880 ok
58880 59392 ok
59392 59904 ok
59904 60416 ok
60416 60928 ok
Sep 22 23:21:47.587 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:21:47.587 INFO Checking if live repair is needed
Sep 22 23:21:47.587 INFO No Live Repair required at this time
60928 61440 ok
61440 61952 ok
61952 62464 ok
62464 62976 ok
62976 63488 ok
63488 64000 ok
64000 64512 ok
64512 65024 ok
65024 65536 ok
65536 66048 ok
66048 66560 ok
66560 67072 ok
67072 67584 ok
67584 68096 ok
68096 68608 ok
68608 69120 ok
69120 69632 ok
69632 70144 ok
70144 70656 ok
70656 71168 ok
71168 71680 ok
71680 72192 ok
72192 72704 ok
72704 73216 ok
73216 73728 ok
73728 74240 ok
74240 74752 ok
74752 75264 ok
75264 75776 ok
75776 76288 ok
76288 76800 ok
76800 77312 ok
77312 77824 ok
77824 78336 ok
78336 78848 ok
78848 79360 ok
79360 79872 ok
79872 80384 ok
80384 80896 ok
80896 81408 ok
81408 81920 ok
81920 82432 ok
82432 82944 ok
82944 83456 ok
83456 83968 ok
83968 84480 ok
84480 84992 ok
84992 85504 ok
85504 86016 ok
86016 86528 ok
86528 87040 ok
87040 87552 ok
87552 88064 ok
88064 88576 ok
88576 89088 ok
89088 89600 ok
89600 90112 ok
90112 90624 ok
90624 91136 ok
91136 91648 ok
91648 92160 ok
92160 92672 ok
92672 93184 ok
93184 93696 ok
93696 94208 ok
94208 94720 ok
94720 95232 ok
95232 95744 ok
95744 96256 ok
96256 96768 ok
96768 97280 ok
97280 97792 ok
97792 98304 ok
98304 98816 ok
98816 99328 ok
99328 99840 ok
99840 100352 ok
100352 100864 ok
100864 101376 ok
101376 101888 ok
101888 102400 ok
102400 102912 ok
102912 103424 ok
103424 103936 ok
103936 104448 ok
104448 104960 ok
104960 105472 ok
105472 105984 ok
105984 106496 ok
106496 107008 ok
107008 107520 ok
107520 108032 ok
108032 108544 ok
108544 109056 ok
109056 109568 ok
109568 110080 ok
110080 110592 ok
110592 111104 ok
111104 111616 ok
111616 112128 ok
112128 112640 ok
112640 113152 ok
113152 113664 ok
113664 114176 ok
114176 114688 ok
114688 115200 ok
115200 115712 ok
115712 116224 ok
116224 116736 ok
116736 117248 ok
117248 117760 ok
117760 118272 ok
118272 118784 ok
118784 119296 ok
119296 119808 ok
119808 120320 ok
120320 120832 ok
120832 121344 ok
121344 121856 ok
121856 122368 ok
122368 122880 ok
122880 123392 ok
123392 123904 ok
123904 124416 ok
124416 124928 ok
124928 125440 ok
125440 125952 ok
125952 126464 ok
{"msg":"[0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.588502975Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] Proc runs for 127.0.0.1:55664 in state Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.588552205Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.588577096Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] Proc runs for 127.0.0.1:37034 in state Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.588601605Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.588631759Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] Proc runs for 127.0.0.1:53040 in state Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.588656691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:47.588 INFO accepted connection from 127.0.0.1:42145, task: main
Sep 22 23:21:47.588 INFO accepted connection from 127.0.0.1:36746, task: main
Sep 22 23:21:47.588 INFO accepted connection from 127.0.0.1:48902, task: main
Sep 22 23:21:47.589 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:5 g:2
Sep 22 23:21:47.589 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:5 g:2
Sep 22 23:21:47.589 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:5 g:2
126464 126976 ok
126976 127488 ok
127488 128000 ok
128000 128512 ok
128512 129024 ok
129024 129536 ok
129536 130048 ok
130048 130560 ok
130560 131072 ok
131072 131584 ok
131584 132096 ok
132096 132608 ok
132608 133120 ok
133120 133632 ok
133632 134144 ok
134144 134656 ok
134656 135168 ok
135168 135680 ok
135680 136192 ok
136192 136704 ok
136704 137216 ok
137216 137728 ok
137728 138240 ok
138240 138752 ok
138752 139264 ok
139264 139776 ok
139776 140288 ok
140288 140800 ok
140800 141312 ok
141312 141824 ok
141824 142336 ok
142336 142848 ok
142848 143360 ok
143360 143872 ok
143872 144384 ok
144384 144896 ok
144896 145408 ok
145408 145920 ok
145920 146432 ok
146432 146944 ok
146944 147456 ok
147456 147968 ok
147968 148480 ok
148480 148992 ok
148992 149504 ok
149504 150016 ok
150016 150528 ok
150528 151040 ok
151040 151552 ok
151552 152064 ok
152064 152576 ok
152576 153088 ok
153088 153600 ok
153600 154112 ok
154112 154624 ok
154624 155136 ok
155136 155648 ok
155648 156160 ok
156160 156672 ok
156672 157184 ok
157184 157696 ok
157696 158208 ok
158208 158720 ok
158720 159232 ok
159232 159744 ok
159744 160256 ok
160256 160768 ok
160768 161280 ok
161280 161792 ok
161792 162304 ok
162304 162816 ok
162816 163328 ok
163328 163840 ok
163840 164352 ok
164352 164864 ok
164864 165376 ok
165376 165888 ok
165888 166400 ok
166400 166912 ok
166912 167424 ok
167424 167936 ok
167936 168448 ok
168448 168960 ok
168960 169472 ok
169472 169984 ok
169984 170496 ok
170496 171008 ok
171008 171520 ok
171520 172032 ok
172032 172544 ok
172544 173056 ok
173056 173568 ok
173568 174080 ok
174080 174592 ok
174592 175104 ok
175104 175616 ok
175616 176128 ok
176128 176640 ok
176640 177152 ok
177152 177664 ok
177664 178176 ok
178176 178688 ok
178688 179200 ok
179200 179712 ok
179712 180224 ok
180224 180736 ok
180736 181248 ok
181248 181760 ok
181760 182272 ok
182272 182784 ok
182784 183296 ok
183296 183808 ok
183808 184320 ok
184320 184832 ok
184832 185344 ok
185344 185856 ok
185856 186368 ok
186368 186880 ok
186880 187392 ok
187392 187904 ok
187904 188416 ok
188416 188928 ok
188928 189440 ok
189440 189952 ok
189952 190464 ok
190464 190976 ok
190976 191488 ok
191488 192000 ok
Sep 22 23:21:47.590 INFO Connection request from 40e06af7-e69f-43e2-974b-bb00dc5e8960 with version 4, task: proc
Sep 22 23:21:47.590 INFO upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:47.590 INFO Connection request from 40e06af7-e69f-43e2-974b-bb00dc5e8960 with version 4, task: proc
Sep 22 23:21:47.590 INFO upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:47.590 INFO Connection request from 40e06af7-e69f-43e2-974b-bb00dc5e8960 with version 4, task: proc
Sep 22 23:21:47.590 INFO upstairs UpstairsConnection { upstairs_id: 40e06af7-e69f-43e2-974b-bb00dc5e8960, session_id: d90ddc5f-b441-432b-bb73-5ab29b73ba5b, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:47.590 DEBG up_ds_listen was notified
Sep 22 23:21:47.590 DEBG up_ds_listen process 1001
Sep 22 23:21:47.590 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:21:47.590 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:21:47.590 DEBG up_ds_listen checked 1 jobs, back to waiting
192000 192512 ok
192512 193024 ok
193024 193536 ok
193536 194048 ok
194048 194560 ok
194560 195072 ok
195072 195584 ok
195584 196096 ok
196096 196608 ok
196608 197120 ok
197120 197632 ok
197632 198144 ok
198144 198656 ok
198656 199168 ok
199168 199680 ok
199680 200192 ok
200192 200704 ok
200704 201216 ok
201216 201728 ok
201728 202240 ok
202240 202752 ok
202752 203264 ok
203264 203776 ok
203776 204288 ok
204288 204800 ok
204800 205312 ok
205312 205824 ok
205824 206336 ok
206336 206848 ok
206848 207360 ok
207360 207872 ok
207872 208384 ok
208384 208896 ok
208896 209408 ok
209408 209920 ok
209920 210432 ok
210432 210944 ok
210944 211456 ok
211456 211968 ok
211968 212480 ok
212480 212992 ok
212992 213504 ok
213504 214016 ok
214016 214528 ok
214528 215040 ok
215040 215552 ok
215552 216064 ok
216064 216576 ok
216576 217088 ok
217088 217600 ok
217600 218112 ok
218112 218624 ok
218624 219136 ok
219136 219648 ok
219648 220160 ok
220160 220672 ok
220672 221184 ok
221184 221696 ok
221696 222208 ok
222208 222720 ok
222720 223232 ok
223232 223744 ok
223744 224256 ok
224256 224768 ok
224768 225280 ok
225280 225792 ok
225792 226304 ok
226304 226816 ok
226816 227328 ok
227328 227840 ok
227840 228352 ok
228352 228864 ok
228864 229376 ok
229376 229888 ok
229888 230400 ok
230400 230912 ok
230912 231424 ok
231424 231936 ok
231936 232448 ok
232448 232960 ok
232960 233472 ok
233472 233984 ok
233984 234496 ok
234496 235008 ok
235008 235520 ok
235520 236032 ok
236032 236544 ok
236544 237056 ok
237056 237568 ok
237568 238080 ok
238080 238592 ok
238592 239104 ok
239104 239616 ok
239616 240128 ok
240128 240640 ok
240640 241152 ok
241152 241664 ok
241664 242176 ok
242176 242688 ok
242688 243200 ok
243200 243712 ok
243712 244224 ok
244224 244736 ok
244736 245248 ok
245248 245760 ok
245760 246272 ok
246272 246784 ok
246784 247296 ok
247296 247808 ok
247808 248320 ok
248320 248832 ok
248832 249344 ok
249344 249856 ok
249856 250368 ok
250368 250880 ok
250880 251392 ok
251392 251904 ok
251904 252416 ok
252416 252928 ok
252928 253440 ok
253440 253952 ok
253952 254464 ok
254464 254976 ok
254976 255488 ok
255488 256000 ok
256000 256512 ok
256512 257024 ok
257024 257536 ok
{"msg":"[0] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) Disconnected Disconnected Disconnected ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.591253352Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from Disconnected to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.591285127Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) WaitActive Disconnected Disconnected ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.591311262Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from Disconnected to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.591333833Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
257536 258048 ok
258048 258560 ok
258560 259072 ok
259072{"msg":"[2] 40e06af7-e69f-43e2-974b-bb00dc5e8960 (d90ddc5f-b441-432b-bb73-5ab29b73ba5b) WaitActive WaitActive Disconnected ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.591358439Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
 259584 ok
259584 {"msg":"[2] Transition from Disconnected to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.591418248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
260096 ok
260096 260608 ok
260608 261120 ok
261120 261632 ok
261632 262144 ok
262144 262656 ok
262656 263168 ok
263168 263680 ok
263680 264192 ok
264192 264704 ok
264704 265216 ok
265216 265728 ok
265728 266240 ok
266240 266752 ok
266752 267264 ok
267264 267776 ok
267776 268288 ok
268288 268800 ok
268800 269312 ok
269312 269824 ok
269824 270336 ok
270336 270848 ok
270848 271360 ok
271360 271872 ok
271872 272384 ok
272384 272896 ok
272896 273408 ok
273408 273920 ok
273920 274432 ok
274432 274944 ok
274944 275456 ok
275456 275968 ok
275968 276480 ok
276480 276992 ok
276992 277504 ok
277504 278016 ok
278016 278528 ok
278528 279040 ok
279040 279552 ok
279552 280064 ok
280064 280576 ok
280576 281088 ok
281088 281600 ok
281600 282112 ok
282112 282624 ok
282624 283136 ok
283136 283648 ok
283648 284160 ok
284160 284672 ok
284672 285184 ok
285184 285696 ok
285696 286208 ok
286208 286720 ok
286720 287232 ok
287232 287744 ok
287744 288256 ok
288256 288768 ok
288768 289280 ok
289280 289792 ok
289792 290304 ok
290304 290816 ok
290816 291328 ok
291328 291840 ok
291840 292352 ok
292352 292864 ok
292864 293376 ok
293376 293888 ok
293888 294400 ok
294400 294912 ok
294912 295424 ok
295424 295936 ok
295936 296448 ok
296448 296960 ok
296960 297472 ok
297472 297984 ok
297984 298496 ok
298496 299008 ok
299008 299520 ok
299520 300032 ok
300032 300544 ok
300544 301056 ok
301056 301568 ok
301568 302080 ok
302080 302592 ok
302592 303104 ok
303104 303616 ok
303616 304128 ok
304128 304640 ok
304640 305152 ok
305152 305664 ok
305664 306176 ok
306176 306688 ok
306688 307200 ok
307200 307712 ok
307712 308224 ok
308224 308736 ok
308736 309248 ok
309248 309760 ok
309760 310272 ok
310272 310784 ok
310784 311296 ok
311296 311808 ok
311808 312320 ok
312320 312832 ok
312832 313344 ok
313344 313856 ok
313856 314368 ok
314368 314880 ok
314880 315392 ok
315392 315904 ok
315904 316416 ok
316416 316928 ok
316928 317440 ok
317440 317952 ok
317952 318464 ok
318464 318976 ok
318976 319488 ok
319488 320000 ok
320000 320512 ok
320512 321024 ok
321024 321536 ok
321536 322048 ok
322048 322560 ok
322560 323072 ok
323072 323584 ok
323584 324096 ok
324096 324608 ok
324608 325120 ok
325120 325632 ok
325632 326144 ok
326144 326656 ok
326656 327168 ok
327168 327680 ok
327680 328192 ok
328192 328704 ok
328704 329216 ok
329216 329728 ok
329728 330240 ok
330240 330752 ok
330752 331264 ok
331264 331776 ok
331776 332288 ok
332288 332800 ok
332800 333312 ok
333312 333824 ok
333824 334336 ok
334336 334848 ok
334848 335360 ok
335360 335872 ok
335872 336384 ok
336384 336896 ok
336896 337408 ok
337408 337920 ok
337920 338432 ok
338432 338944 ok
338944 339456 ok
339456 339968 ok
339968 340480 ok
340480 340992 ok
340992 341504 ok
341504 342016 ok
342016 342528 ok
342528 343040 ok
343040 343552 ok
343552 344064 ok
344064 344576 ok
344576 345088 ok
345088 345600 ok
345600 346112 ok
346112 346624 ok
346624 347136 ok
347136 347648 ok
347648 348160 ok
348160 348672 ok
348672 349184 ok
349184 349696 ok
349696 350208 ok
350208 350720 ok
350720 351232 ok
351232 351744 ok
351744 352256 ok
352256 352768 ok
352768 353280 ok
353280 353792 ok
353792 354304 ok
354304 354816 ok
354816 355328 ok
355328 355840 ok
355840 356352 ok
356352 356864 ok
356864 357376 ok
357376 357888 ok
357888 358400 ok
358400 358912 ok
358912 359424 ok
359424 359936 ok
359936 360448 ok
360448 360960 ok
360960 361472 ok
361472 361984 ok
361984 362496 ok
362496 363008 ok
363008 363520 ok
363520 364032 ok
364032 364544 ok
364544 365056 ok
365056 365568 ok
365568 366080 ok
366080 366592 ok
366592 367104 ok
367104 367616 ok
367616 368128 ok
368128 368640 ok
368640 369152 ok
369152 369664 ok
369664 370176 ok
370176 370688 ok
370688 371200 ok
371200 371712 ok
371712 372224 ok
372224 372736 ok
372736 373248 ok
373248 373760 ok
373760 374272 ok
374272 374784 ok
374784 375296 ok
375296 375808 ok
375808 376320 ok
376320 376832 ok
376832 377344 ok
377344 377856 ok
377856 378368 ok
378368 378880 ok
378880 379392 ok
379392 379904 ok
379904 380416 ok
380416 380928 ok
380928 381440 ok
381440 381952 ok
381952 382464 ok
382464 382976 ok
382976 383488 ok
383488 384000 ok
384000 384512 ok
384512 385024 ok
385024 385536 ok
385536 386048 ok
386048 386560 ok
386560 387072 ok
387072 387584 ok
387584 388096 ok
388096 388608 ok
388608 389120 ok
389120 389632 ok
389632 390144 ok
390144 390656 ok
390656 391168 ok
391168 391680 ok
391680 392192 ok
392192 392704 ok
392704 393216 ok
393216 393728 ok
393728 394240 ok
394240 394752 ok
394752 395264 ok
395264 395776 ok
395776 396288 ok
396288 396800 ok
396800 397312 ok
397312 397824 ok
397824 398336 ok
398336 398848 ok
398848 399360 ok
399360 399872 ok
399872 400384 ok
400384 400896 ok
400896 401408 ok
401408 401920 ok
401920 402432 ok
402432 402944 ok
402944 403456 ok
403456 403968 ok
403968 404480 ok
404480 404992 ok
404992 405504 ok
405504 406016 ok
406016 406528 ok
406528 407040 ok
407040 407552 ok
407552 408064 ok
408064 408576 ok
408576 409088 ok
409088 409600 ok
409600 410112 ok
410112 410624 ok
410624 411136 ok
411136 411648 ok
411648 412160 ok
412160 412672 ok
412672 413184 ok
413184 413696 ok
413696 414208 ok
414208 414720 ok
414720 415232 ok
415232 415744 ok
415744 416256 ok
416256 416768 ok
416768 417280 ok
417280 417792 ok
417792 418304 ok
418304 418816 ok
418816 419328 ok
419328 419840 ok
419840 420352 ok
420352 420864 ok
420864 421376 ok
421376 421888 ok
421888 422400 ok
422400 422912 ok
422912 423424 ok
423424 423936 ok
423936 424448 ok
424448 424960 ok
424960 425472 ok
425472 425984 ok
425984 426496 ok
426496 427008 ok
427008 427520 ok
427520 428032 ok
428032 428544 ok
428544 429056 ok
429056 429568 ok
429568 430080 ok
430080 430592 ok
430592 431104 ok
431104 431616 ok
431616 432128 ok
432128 432640 ok
432640 433152 ok
433152 433664 ok
433664 434176 ok
434176 434688 ok
434688 435200 ok
435200 435712 ok
435712 436224 ok
436224 436736 ok
436736 437248 ok
437248 437760 ok
437760 438272 ok
438272 438784 ok
438784 439296 ok
439296 439808 ok
439808 440320 ok
440320 440832 ok
440832 441344 ok
441344 441856 ok
441856 442368 ok
442368 442880 ok
442880 443392 ok
443392 443904 ok
443904 444416 ok
444416 444928 ok
444928 445440 ok
445440 445952 ok
445952 446464 ok
446464 446976 ok
446976 447488 ok
447488 448000 ok
448000 448512 ok
448512 449024 ok
449024 449536 ok
449536 450048 ok
450048 450560 ok
450560 451072 ok
451072 451584 ok
451584 452096 ok
452096 452608 ok
452608 453120 ok
453120 453632 ok
453632 454144 ok
454144 454656 ok
454656 455168 ok
455168 455680 ok
455680 456192 ok
456192 456704 ok
456704 457216 ok
457216 457728 ok
457728 458240 ok
458240 458752 ok
458752 459264 ok
459264 459776 ok
459776 460288 ok
460288 460800 ok
460800 461312 ok
461312 461824 ok
461824 462336 ok
462336 462848 ok
462848 463360 ok
463360 463872 ok
463872 464384 ok
464384 464896 ok
464896 465408 ok
465408 465920 ok
465920 466432 ok
466432 466944 ok
466944 467456 ok
467456 467968 ok
467968 468480 ok
468480 468992 ok
468992 469504 ok
469504 470016 ok
470016 470528 ok
470528 471040 ok
471040 471552 ok
471552 472064 ok
472064 472576 ok
472576 473088 ok
473088 473600 ok
473600 474112 ok
474112 474624 ok
474624 475136 ok
475136 475648 ok
475648 476160 ok
476160 476672 ok
476672 477184 ok
477184 477696 ok
477696 478208 ok
478208 478720 ok
478720 479232 ok
479232 479744 ok
479744 480256 ok
480256 480768 ok
480768 481280 ok
481280 481792 ok
481792 482304 ok
482304 482816 ok
482816 483328 ok
483328 483840 ok
483840 484352 ok
484352 484864 ok
484864 485376 ok
485376 485888 ok
485888 486400 ok
486400 486912 ok
486912 487424 ok
487424 487936 ok
487936 488448 ok
488448 488960 ok
488960 489472 ok
489472 489984 ok
489984 490496 ok
490496 491008 ok
491008 491520 ok
491520 492032 ok
492032 492544 ok
492544 493056 ok
493056 493568 ok
493568 494080 ok
494080 494592 ok
494592 495104 ok
495104 495616 ok
495616 496128 ok
496128 496640 ok
496640 497152 ok
497152 497664 ok
497664 498176 ok
498176 498688 ok
498688 499200 ok
499200 499712 ok
499712 500224 ok
500224 500736 ok
500736 501248 ok
501248 501760 ok
501760 502272 ok
502272 502784 ok
502784 503296 ok
503296 503808 ok
503808 504320 ok
504320 504832 ok
504832 505344 ok
505344 505856 ok
505856 506368 ok
506368 506880 ok
506880 507392 ok
507392 507904 ok
507904 508416 ok
508416 508928 ok
508928 509440 ok
509440 509952 ok
509952 510464 ok
510464 510976 ok
510976 511488 ok
511488 512000 ok
512000 512512 ok
512512 513024 ok
513024 513536 ok
513536 514048 ok
514048 514560 ok
514560 515072 ok
515072 515584 ok
515584 516096 ok
516096 516608 ok
516608 517120 ok
517120 517632 ok
517632 518144 ok
518144 518656 ok
518656 519168 ok
519168 519680 ok
519680 520192 ok
520192 520704 ok
520704 521216 ok
521216 521728 ok
521728 522240 ok
522240 522752 ok
522752 523264 ok
523264 523776 ok
523776 524288 ok
524288 524800 ok
524800 525312 ok
525312 525824 ok
525824 526336 ok
526336 526848 ok
526848 527360 ok
527360 527872 ok
527872 528384 ok
528384 528896 ok
528896 529408 ok
529408 529920 ok
529920 530432 ok
530432 530944 ok
530944 531456 ok
531456 531968 ok
531968 532480 ok
532480 532992 ok
532992 533504 ok
533504 534016 ok
534016 534528 ok
534528 535040 ok
535040 535552 ok
535552 536064 ok
536064 536576 ok
536576 537088 ok
537088 537600 ok
537600 538112 ok
538112 538624 ok
538624 539136 ok
539136 539648 ok
539648 540160 ok
540160 540672 ok
540672 541184 ok
541184 541696 ok
541696 542208 ok
542208 542720 ok
542720 543232 ok
543232 543744 ok
543744 544256 ok
544256 544768 ok
544768 545280 ok
545280 545792 ok
545792 546304 ok
546304 546816 ok
546816 547328 ok
547328 547840 ok
547840 548352 ok
548352 548864 ok
548864 549376 ok
549376 549888 ok
549888 550400 ok
550400 550912 ok
550912 551424 ok
551424 551936 ok
551936 552448 ok
552448 552960 ok
552960 553472 ok
553472 553984 ok
553984 554496 ok
554496 555008 ok
555008 555520 ok
555520 556032 ok
556032 556544 ok
556544 557056 ok
557056 557568 ok
557568 558080 ok
558080 558592 ok
558592 559104 ok
559104 559616 ok
559616 560128 ok
560128 560640 ok
560640 561152 ok
561152 561664 ok
561664 562176 ok
562176 562688 ok
562688 563200 ok
563200 563712 ok
563712 564224 ok
564224 564736 ok
564736 565248 ok
565248 565760 ok
565760 566272 ok
566272 566784 ok
566784 567296 ok
567296 567808 ok
567808 568320 ok
568320 568832 ok
568832 569344 ok
569344 569856 ok
569856 570368 ok
570368 570880 ok
570880 571392 ok
571392 571904 ok
571904 572416 ok
572416 572928 ok
572928 573440 ok
573440 573952 ok
573952 574464 ok
574464 574976 ok
574976 575488 ok
575488 576000 ok
576000 576512 ok
576512 577024 ok
577024 577536 ok
577536 578048 ok
578048 578560 ok
578560 579072 ok
579072 579584 ok
579584 580096 ok
580096 580608 ok
580608 581120 ok
581120 581632 ok
581632 582144 ok
582144 582656 ok
582656 583168 ok
583168 583680 ok
583680 584192 ok
584192 584704 ok
584704 585216 ok
585216 585728 ok
585728 586240 ok
586240 586752 ok
586752 587264 ok
587264 587776 ok
587776 588288 ok
588288 588800 ok
588800 589312 ok
589312 589824 ok
589824 590336 ok
590336 590848 ok
590848 591360 ok
591360 591872 ok
591872 592384 ok
592384 592896 ok
592896 593408 ok
593408 593920 ok
593920 594432 ok
594432 594944 ok
594944 595456 ok
595456 595968 ok
595968 596480 ok
596480 596992 ok
596992 597504 ok
597504 598016 ok
598016 598528 ok
598528 599040 ok
599040 599552 ok
599552 600064 ok
600064 600576 ok
600576 601088 ok
601088 601600 ok
601600 602112 ok
602112 602624 ok
602624 603136 ok
603136 603648 ok
603648 604160 ok
604160 604672 ok
604672 605184 ok
605184 605696 ok
605696 606208 ok
606208 606720 ok
606720 607232 ok
607232 607744 ok
607744 608256 ok
608256 608768 ok
608768 609280 ok
609280 609792 ok
609792 610304 ok
610304 610816 ok
610816 611328 ok
611328 611840 ok
611840 612352 ok
612352 612864 ok
612864 613376 ok
613376 613888 ok
613888 614400 ok
614400 614912 ok
614912 615424 ok
615424 615936 ok
615936 616448 ok
616448 616960 ok
616960 617472 ok
617472 617984 ok
617984 618496 ok
618496 619008 ok
619008 619520 ok
619520 620032 ok
620032 620544 ok
620544 621056 ok
621056 621568 ok
621568 622080 ok
622080 622592 ok
622592 623104 ok
623104 623616 ok
623616 624128 ok
624128 624640 ok
624640 625152 ok
625152 625664 ok
625664 626176 ok
626176 626688 ok
626688 627200 ok
627200 627712 ok
627712 628224 ok
628224 628736 ok
628736 629248 ok
629248 629760 ok
629760 630272 ok
630272 630784 ok
630784 631296 ok
631296 631808 ok
631808 632320 ok
632320 632832 ok
632832 633344 ok
633344 633856 ok
633856 634368 ok
634368 634880 ok
634880 635392 ok
635392 635904 ok
635904 636416 ok
636416 636928 ok
636928 637440 ok
637440 637952 ok
637952 638464 ok
638464 638976 ok
638976 639488 ok
639488 640000 ok
640000 640512 ok
640512 641024 ok
641024 641536 ok
641536 642048 ok
642048 642560 ok
642560 643072 ok
643072 643584 ok
643584 644096 ok
644096 644608 ok
644608 645120 ok
645120 645632 ok
645632 646144 ok
646144 646656 ok
646656 647168 ok
647168 647680 ok
647680 648192 ok
648192 648704 ok
648704 649216 ok
649216 649728 ok
649728 650240 ok
650240 650752 ok
650752 651264 ok
651264 651776 ok
651776 652288 ok
652288 652800 ok
652800 653312 ok
653312 653824 ok
653824 654336 ok
654336 654848 ok
654848 655360 ok
655360 655872 ok
655872 656384 ok
656384 656896 ok
656896 657408 ok
657408 657920 ok
657920 658432 ok
658432 658944 ok
658944 659456 ok
659456 659968 ok
659968 660480 ok
660480 660992 ok
660992 661504 ok
661504 662016 ok
662016 662528 ok
662528 663040 ok
663040 663552 ok
663552 664064 ok
664064 664576 ok
664576 665088 ok
665088 665600 ok
665600 666112 ok
666112 666624 ok
666624 667136 ok
667136 667648 ok
667648 668160 ok
668160 668672 ok
668672 669184 ok
669184 669696 ok
669696 670208 ok
670208 670720 ok
670720 671232 ok
671232 671744 ok
671744 672256 ok
672256 672768 ok
672768 673280 ok
673280 673792 ok
673792 674304 ok
674304 674816 ok
674816 675328 ok
675328 675840 ok
675840 676352 ok
676352 676864 ok
676864 677376 ok
677376 677888 ok
677888 678400 ok
678400 678912 ok
678912 679424 ok
679424 679936 ok
679936 680448 ok
680448 680960 ok
680960 681472 ok
681472 681984 ok
681984 682496 ok
682496 683008 ok
683008 683520 ok
683520 684032 ok
684032 684544 ok
684544 685056 ok
685056 685568 ok
685568 686080 ok
686080 686592 ok
686592 687104 ok
687104 687616 ok
687616 688128 ok
688128 688640 ok
688640 689152 ok
689152 689664 ok
689664 690176 ok
690176 690688 ok
690688 691200 ok
691200 691712 ok
691712 692224 ok
692224 692736 ok
692736 693248 ok
693248 693760 ok
693760 694272 ok
694272 694784 ok
694784 695296 ok
695296 695808 ok
695808 696320 ok
696320 696832 ok
696832 697344 ok
697344 697856 ok
697856 698368 ok
698368 698880 ok
698880 699392 ok
699392 699904 ok
699904 700416 ok
700416 700928 ok
700928 701440 ok
701440 701952 ok
701952 702464 ok
702464 702976 ok
702976 703488 ok
703488 704000 ok
704000 704512 ok
704512 705024 ok
705024 705536 ok
705536 706048 ok
706048 706560 ok
706560 707072 ok
707072 707584 ok
707584 708096 ok
708096 708608 ok
708608 709120 ok
709120 709632 ok
709632 710144 ok
710144 710656 ok
710656 711168 ok
711168 711680 ok
711680 712192 ok
712192 712704 ok
712704 713216 ok
713216 713728 ok
713728 714240 ok
714240 714752 ok
714752 715264 ok
715264 715776 ok
715776 716288 ok
716288 716800 ok
716800 717312 ok
717312 717824 ok
717824 718336 ok
718336 718848 ok
718848 719360 ok
719360 719872 ok
719872 720384 ok
720384 720896 ok
720896 721408 ok
721408 721920 ok
721920 722432 ok
722432 722944 ok
722944 723456 ok
723456 723968 ok
723968 724480 ok
724480 724992 ok
724992 725504 ok
725504 726016 ok
726016 726528 ok
726528 727040 ok
727040 727552 ok
727552 728064 ok
728064 728576 ok
728576 729088 ok
729088 729600 ok
729600 730112 ok
730112 730624 ok
730624 731136 ok
731136 731648 ok
731648 732160 ok
732160 732672 ok
732672 733184 ok
733184 733696 ok
733696 734208 ok
734208 734720 ok
734720 735232 ok
735232 735744 ok
735744 736256 ok
736256 736768 ok
736768 737280 ok
737280 737792 ok
737792 738304 ok
738304 738816 ok
738816 739328 ok
739328 739840 ok
739840 740352 ok
740352 740864 ok
740864 741376 ok
741376 741888 ok
741888 742400 ok
742400 742912 ok
742912 743424 ok
743424 743936 ok
743936 744448 ok
744448 744960 ok
744960 745472 ok
745472 745984 ok
745984 746496 ok
746496 747008 ok
747008 747520 ok
747520 748032 ok
748032 748544 ok
748544 749056 ok
749056 749568 ok
749568 750080 ok
750080 750592 ok
750592 751104 ok
751104 751616 ok
751616 752128 ok
752128 752640 ok
752640 753152 ok
753152 753664 ok
753664 754176 ok
754176 754688 ok
754688 755200 ok
755200 755712 ok
755712 756224 ok
756224 756736 ok
756736 757248 ok
757248 757760 ok
757760 758272 ok
758272 758784 ok
758784 759296 ok
759296 759808 ok
759808 760320 ok
760320 760832 ok
760832 761344 ok
761344 761856 ok
761856 762368 ok
762368 762880 ok
762880 763392 ok
763392 763904 ok
763904 764416 ok
764416 764928 ok
764928 765440 ok
765440 765952 ok
765952 766464 ok
766464 766976 ok
766976 767488 ok
767488 768000 ok
768000 768512 ok
768512 769024 ok
769024 769536 ok
769536 770048 ok
770048 770560 ok
770560 771072 ok
771072 771584 ok
771584 772096 ok
772096 772608 ok
772608 773120 ok
773120 773632 ok
773632 774144 ok
774144 774656 ok
774656 775168 ok
775168 775680 ok
775680 776192 ok
776192 776704 ok
776704 777216 ok
777216 777728 ok
777728 778240 ok
778240 778752 ok
778752 779264 ok
779264 779776 ok
779776 780288 ok
780288 780800 ok
780800 781312 ok
781312 781824 ok
781824 782336 ok
782336 782848 ok
782848 783360 ok
783360 783872 ok
783872 784384 ok
784384 784896 ok
784896 785408 ok
785408 785920 ok
785920 786432 ok
786432 786944 ok
786944 787456 ok
787456 787968 ok
787968 788480 ok
788480 788992 ok
788992 789504 ok
789504 790016 ok
790016 790528 ok
790528 791040 ok
791040 791552 ok
791552 792064 ok
792064 792576 ok
792576 793088 ok
793088 793600 ok
793600 794112 ok
794112 794624 ok
794624 795136 ok
795136 795648 ok
795648 796160 ok
796160 796672 ok
796672 797184 ok
797184 797696 ok
797696 798208 ok
798208 798720 ok
798720 799232 ok
799232 799744 ok
799744 800256 ok
800256 800768 ok
800768 801280 ok
801280 801792 ok
801792 802304 ok
802304 802816 ok
802816 803328 ok
803328 803840 ok
803840 804352 ok
804352 804864 ok
804864 805376 ok
805376 805888 ok
805888 806400 ok
806400 806912 ok
806912 807424 ok
807424 807936 ok
807936 808448 ok
808448 808960 ok
808960 809472 ok
809472 809984 ok
809984 810496 ok
810496 811008 ok
811008 811520 ok
811520 812032 ok
812032 812544 ok
812544 813056 ok
813056 813568 ok
813568 814080 ok
814080 814592 ok
814592 815104 ok
815104 815616 ok
815616 816128 ok
816128 816640 ok
816640 817152 ok
817152 817664 ok
817664 818176 ok
818176 818688 ok
818688 819200 ok
819200 819712 ok
819712 820224 ok
820224 820736 ok
820736 821248 ok
821248 821760 ok
821760 822272 ok
822272 822784 ok
822784 823296 ok
823296 823808 ok
823808 824320 ok
824320 824832 ok
824832 825344 ok
825344 825856 ok
825856 826368 ok
826368 826880 ok
826880 827392 ok
827392 827904 ok
827904 828416 ok
828416 828928 ok
828928 829440 ok
829440 829952 ok
829952 830464 ok
830464 830976 ok
830976 831488 ok
831488 832000 ok
832000 832512 ok
832512 833024 ok
833024 833536 ok
833536 834048 ok
834048 834560 ok
834560 835072 ok
835072 835584 ok
835584 836096 ok
836096 836608 ok
836608 837120 ok
837120 837632 ok
837632 838144 ok
838144 838656 ok
838656 839168 ok
839168 839680 ok
839680 840192 ok
840192 840704 ok
840704 841216 ok
841216 841728 ok
841728 842240 ok
842240 842752 ok
842752 843264 ok
843264 843776 ok
843776 844288 ok
844288 844800 ok
844800 845312 ok
845312 845824 ok
845824 846336 ok
846336 846848 ok
846848 847360 ok
847360 847872 ok
847872 848384 ok
848384 848896 ok
848896 849408 ok
849408 849920 ok
849920 850432 ok
850432 850944 ok
850944 851456 ok
851456 851968 ok
851968 852480 ok
852480 852992 ok
852992 853504 ok
853504 854016 ok
854016 854528 ok
854528 855040 ok
855040 855552 ok
855552 856064 ok
856064 856576 ok
856576 857088 ok
857088 857600 ok
857600 858112 ok
858112 858624 ok
858624 859136 ok
859136 859648 ok
859648 860160 ok
860160 860672 ok
860672 861184 ok
861184 861696 ok
861696 862208 ok
862208 862720 ok
862720 863232 ok
863232 863744 ok
863744 864256 ok
864256 864768 ok
864768 865280 ok
865280 865792 ok
865792 866304 ok
866304 866816 ok
866816 867328 ok
867328 867840 ok
867840 868352 ok
868352 868864 ok
868864 869376 ok
869376 869888 ok
869888 870400 ok
870400 870912 ok
870912 871424 ok
871424 871936 ok
871936 872448 ok
872448 872960 ok
872960 873472 ok
873472 873984 ok
873984 874496 ok
874496 875008 ok
875008 875520 ok
875520 876032 ok
876032 876544 ok
876544 877056 ok
877056 877568 ok
877568 878080 ok
878080 878592 ok
878592 879104 ok
879104 879616 ok
879616 880128 ok
880128 880640 ok
880640 881152 ok
881152 881664 ok
881664 882176 ok
882176 882688 ok
882688 883200 ok
883200 883712 ok
883712 884224 ok
884224 884736 ok
884736 885248 ok
885248 885760 ok
885760 886272 ok
886272 886784 ok
886784 887296 ok
887296 887808 ok
887808 888320 ok
888320 888832 ok
888832 889344 ok
889344 889856 ok
889856 890368 ok
890368 890880 ok
890880 891392 ok
891392 891904 ok
891904 892416 ok
892416 892928 ok
892928 893440 ok
893440 893952 ok
893952 894464 ok
894464 894976 ok
894976 895488 ok
895488 896000 ok
896000 896512 ok
896512 897024 ok
897024 897536 ok
897536 898048 ok
898048 898560 ok
898560 899072 ok
899072 899584 ok
899584 900096 ok
900096 900608 ok
900608 901120 ok
901120 901632 ok
901632 902144 ok
902144 902656 ok
902656 903168 ok
903168 903680 ok
903680 904192 ok
904192 904704 ok
904704 905216 ok
905216 905728 ok
905728 906240 ok
906240 906752 ok
906752 907264 ok
907264 907776 ok
907776 908288 ok
908288 908800 ok
908800 909312 ok
909312 909824 ok
909824 910336 ok
910336 910848 ok
910848 911360 ok
911360 911872 ok
911872 912384 ok
912384 912896 ok
912896 913408 ok
913408 913920 ok
913920 914432 ok
914432 914944 ok
914944 915456 ok
915456 915968 ok
915968 916480 ok
916480 916992 ok
916992 917504 ok
917504 918016 ok
918016 918528 ok
918528 919040 ok
919040 919552 ok
919552 920064 ok
920064 920576 ok
920576 921088 ok
921088 921600 ok
921600 922112 ok
922112 922624 ok
922624 923136 ok
923136 923648 ok
923648 924160 ok
924160 924672 ok
924672 925184 ok
925184 925696 ok
925696 926208 ok
926208 926720 ok
926720 927232 ok
927232 927744 ok
927744 928256 ok
928256 928768 ok
928768 929280 ok
929280 929792 ok
929792 930304 ok
930304 930816 ok
930816 931328 ok
931328 931840 ok
931840 932352 ok
932352 932864 ok
932864 933376 ok
933376 933888 ok
933888 934400 ok
934400 934912 ok
934912 935424 ok
935424 935936 ok
935936 936448 ok
936448 936960 ok
936960 937472 ok
937472 937984 ok
937984 938496 ok
938496 939008 ok
939008 939520 ok
939520 940032 ok
940032 940544 ok
940544 941056 ok
941056 941568 ok
941568 942080 ok
942080 942592 ok
942592 943104 ok
943104 943616 ok
943616 944128 ok
944128 944640 ok
944640 945152 ok
945152 945664 ok
945664 946176 ok
946176 946688 ok
946688 947200 ok
947200 947712 ok
947712 948224 ok
948224 948736 ok
948736 949248 ok
949248 949760 ok
949760 950272 ok
950272 950784 ok
950784 951296 ok
951296 951808 ok
951808 952320 ok
952320 952832 ok
952832 953344 ok
953344 953856 ok
953856 954368 ok
954368 954880 ok
954880 955392 ok
955392 955904 ok
955904 956416 ok
956416 956928 ok
956928 957440 ok
957440 957952 ok
957952 958464 ok
958464 958976 ok
958976 959488 ok
959488 960000 ok
960000 960512 ok
960512 961024 ok
961024 961536 ok
961536 962048 ok
962048 962560 ok
962560 963072 ok
963072 963584 ok
963584 964096 ok
964096 964608 ok
964608 965120 ok
965120 965632 ok
965632 966144 ok
966144 966656 ok
966656 967168 ok
967168 967680 ok
967680 968192 ok
968192 968704 ok
968704 969216 ok
969216 969728 ok
969728 970240 ok
970240 970752 ok
970752 971264 ok
971264 971776 ok
971776 972288 ok
972288 972800 ok
972800 973312 ok
973312 973824 ok
973824 974336 ok
974336 974848 ok
974848 975360 ok
975360 975872 ok
975872 976384 ok
976384 976896 ok
976896 977408 ok
977408 977920 ok
977920 978432 ok
978432 978944 ok
978944 979456 ok
979456 979968 ok
979968 980480 ok
980480 980992 ok
980992 981504 ok
981504 982016 ok
982016 982528 ok
982528 983040 ok
983040 983552 ok
983552 984064 ok
984064 984576 ok
984576 985088 ok
985088 985600 ok
985600 986112 ok
986112 986624 ok
986624 987136 ok
987136 987648 ok
987648 988160 ok
988160 988672 ok
988672 989184 ok
989184 989696 ok
989696 990208 ok
990208 990720 ok
990720 991232 ok
991232 991744 ok
991744 992256 ok
992256 992768 ok
992768 993280 ok
993280 993792 ok
993792 994304 ok
994304 994816 ok
994816 995328 ok
995328 995840 ok
995840 996352 ok
996352 996864 ok
996864 997376 ok
997376 997888 ok
997888 998400 ok
998400 998912 ok
998912 999424 ok
999424 999936 ok
999936 1000448 ok
1000448 1000960 ok
1000960 1001472 ok
1001472 1001984 ok
1001984 1002496 ok
1002496 1003008 ok
1003008 1003520 ok
1003520 1004032 ok
1004032 1004544 ok
1004544 1005056 ok
1005056 1005568 ok
1005568 1006080 ok
1006080 1006592 ok
1006592 1007104 ok
1007104 1007616 ok
1007616 1008128 ok
1008128 1008640 ok
1008640 1009152 ok
1009152 1009664 ok
1009664 1010176 ok
1010176 1010688 ok
1010688 1011200 ok
1011200 1011712 ok
1011712 1012224 ok
1012224 1012736 ok
1012736 1013248 ok
1013248 1013760 ok
1013760 1014272 ok
1014272 1014784 ok
1014784 1015296 ok
1015296 1015808 ok
1015808 1016320 ok
1016320 1016832 ok
1016832 1017344 ok
1017344 1017856 ok
1017856 1018368 ok
1018368 1018880 ok
1018880 1019392 ok
1019392 1019904 ok
1019904 1020416 ok
1020416 1020928 ok
1020928 1021440 ok
1021440 1021952 ok
1021952 1022464 ok
1022464 1022976 ok
1022976 1023488 ok
1023488 1024000 ok
1024000 1024512 ok
1024512 1025024 ok
1025024 1025536 ok
1025536 1026048 ok
1026048 1026560 ok
1026560 1027072 ok
1027072 1027584 ok
1027584 1028096 ok
1028096 1028608 ok
1028608 1029120 ok
1029120 1029632 ok
1029632 1030144 ok
1030144 1030656 ok
1030656 1031168 ok
1031168 1031680 ok
1031680 1032192 ok
1032192 1032704 ok
1032704 1033216 ok
1033216 1033728 ok
1033728 1034240 ok
1034240 1034752 ok
1034752 1035264 ok
1035264 1035776 ok
1035776 1036288 ok
1036288 1036800 ok
1036800 1037312 ok
1037312 1037824 ok
1037824 1038336 ok
1038336 1038848 ok
1038848 1039360 ok
1039360 1039872 ok
1039872 1040384 ok
1040384 1040896 ok
1040896 1041408 ok
1041408 1041920 ok
1041920 1042432 ok
1042432 1042944 ok
1042944 1043456 ok
1043456 1043968 ok
1043968 1044480 ok
1044480 1044992 ok
1044992 1045504 ok
1045504 1046016 ok
1046016 1046528 ok
1046528 1047040 ok
1047040 1047552 ok
1047552 1048064 ok
1048064 1048576 ok
1048576 1049088 ok
1049088 1049600 ok
1049600 1050112 ok
1050112 1050624 ok
1050624 1051136 ok
1051136 1051648 ok
1051648 1052160 ok
1052160 1052672 ok
1052672 1053184 ok
1053184 1053696 ok
1053696 1054208 ok
1054208 1054720 ok
1054720 1055232 ok
1055232 1055744 ok
1055744 1056256 ok
1056256 1056768 ok
1056768 1057280 ok
1057280 1057792 ok
1057792 1058304 ok
1058304 1058816 ok
1058816 1059328 ok
1059328 1059840 ok
1059840 1060352 ok
1060352 1060864 ok
1060864 1061376 ok
1061376 1061888 ok
1061888 1062400 ok
1062400 1062912 ok
1062912 1063424 ok
1063424 1063936 ok
1063936 1064448 ok
1064448 1064960 ok
1064960 1065472 ok
1065472 1065984 ok
1065984 1066496 ok
1066496 1067008 ok
1067008 1067520 ok
1067520 1068032 ok
1068032 1068544 ok
1068544 1069056 ok
1069056 1069568 ok
1069568 1070080 ok
1070080 1070592 ok
1070592 1071104 ok
1071104 1071616 ok
1071616 1072128 ok
1072128 1072640 ok
1072640 1073152 ok
1073152 1073664 ok
1073664 1074176 ok
1074176 1074688 ok
1074688 1075200 ok
1075200 1075712 ok
1075712 1076224 ok
1076224 1076736 ok
1076736 1077248 ok
1077248 1077760 ok
1077760 1078272 ok
1078272 1078784 ok
1078784 1079296 ok
1079296 1079808 ok
1079808 1080320 ok
1080320 1080832 ok
1080832 1081344 ok
1081344 1081856 ok
1081856 1082368 ok
1082368 1082880 ok
1082880 1083392 ok
1083392 1083904 ok
1083904 1084416 ok
1084416 1084928 ok
1084928 1085440 ok
1085440 1085952 ok
1085952 1086464 ok
1086464 1086976 ok
1086976 1087488 ok
1087488 1088000 ok
1088000 1088512 ok
1088512 1089024 ok
1089024 1089536 ok
1089536 1090048 ok
1090048 1090560 ok
1090560 1091072 ok
1091072 1091584 ok
1091584 1092096 ok
1092096 1092608 ok
1092608 1093120 ok
1093120 1093632 ok
1093632 1094144 ok
1094144 1094656 ok
1094656 1095168 ok
1095168 1095680 ok
1095680 1096192 ok
1096192 1096704 ok
1096704 1097216 ok
1097216 1097728 ok
1097728 1098240 ok
1098240 1098752 ok
1098752 1099264 ok
1099264 1099776 ok
1099776 1100288 ok
1100288 1100800 ok
1100800 1101312 ok
1101312 1101824 ok
1101824 1102336 ok
1102336 1102848 ok
1102848 1103360 ok
1103360 1103872 ok
1103872 1104384 ok
1104384 1104896 ok
1104896 1105408 ok
1105408 1105920 ok
1105920 1106432 ok
1106432 1106944 ok
1106944 1107456 ok
1107456 1107968 ok
1107968 1108480 ok
1108480 1108992 ok
1108992 1109504 ok
1109504 1110016 ok
1110016 1110528 ok
1110528 1111040 ok
1111040 1111552 ok
1111552 1112064 ok
1112064 1112576 ok
1112576 1113088 ok
1113088 1113600 ok
1113600 1114112 ok
1114112 1114624 ok
1114624 1115136 ok
1115136 1115648 ok
1115648 1116160 ok
1116160 1116672 ok
1116672 1117184 ok
1117184 1117696 ok
1117696 1118208 ok
1118208 1118720 ok
1118720 1119232 ok
1119232 1119744 ok
1119744 1120256 ok
1120256 1120768 ok
1120768 1121280 ok
1121280 1121792 ok
1121792 1122304 ok
1122304 1122816 ok
1122816 1123328 ok
1123328 1123840 ok
1123840 1124352 ok
1124352 1124864 ok
1124864 1125376 ok
1125376 1125888 ok
1125888 1126400 ok
1126400 1126912 ok
1126912 1127424 ok
1127424 1127936 ok
1127936 1128448 ok
1128448 1128960 ok
1128960 1129472 ok
1129472 1129984 ok
1129984 1130496 ok
1130496 1131008 ok
1131008 1131520 ok
1131520 1132032 ok
1132032 1132544 ok
1132544 1133056 ok
1133056 1133568 ok
1133568 1134080 ok
1134080 1134592 ok
1134592 1135104 ok
1135104 1135616 ok
1135616 1136128 ok
1136128 1136640 ok
1136640 1137152 ok
1137152 1137664 ok
1137664 1138176 ok
1138176 1138688 ok
1138688 1139200 ok
1139200 1139712 ok
1139712 1140224 ok
1140224 1140736 ok
1140736 1141248 ok
1141248 1141760 ok
1141760 1142272 ok
1142272 1142784 ok
1142784 1143296 ok
1143296 1143808 ok
1143808 1144320 ok
1144320 1144832 ok
1144832 1145344 ok
1145344 1145856 ok
1145856 1146368 ok
1146368 1146880 ok
1146880 1147392 ok
1147392 1147904 ok
1147904 1148416 ok
1148416 1148928 ok
1148928 1149440 ok
1149440 1149952 ok
1149952 1150464 ok
1150464 1150976 ok
1150976 1151488 ok
1151488 1152000 ok
1152000 1152512 ok
1152512 1153024 ok
1153024 1153536 ok
1153536 1154048 ok
1154048 1154560 ok
1154560 1155072 ok
1155072 1155584 ok
1155584 1156096 ok
1156096 1156608 ok
1156608 1157120 ok
1157120 1157632 ok
1157632 1158144 ok
1158144 1158656 ok
1158656 1159168 ok
1159168 1159680 ok
1159680 1160192 ok
1160192 1160704 ok
1160704 1161216 ok
1161216 1161728 ok
1161728 1162240 ok
1162240 1162752 ok
1162752 1163264 ok
1163264 1163776 ok
1163776 1164288 ok
1164288 1164800 ok
1164800 1165312 ok
1165312 1165824 ok
1165824 1166336 ok
1166336 1166848 ok
1166848 1167360 ok
1167360 1167872 ok
1167872 1168384 ok
1168384 1168896 ok
1168896 1169408 ok
1169408 1169920 ok
1169920 1170432 ok
1170432 1170944 ok
1170944 1171456 ok
1171456 1171968 ok
1171968 1172480 ok
1172480 1172992 ok
1172992 1173504 ok
1173504 1174016 ok
1174016 1174528 ok
1174528 1175040 ok
1175040 1175552 ok
1175552 1176064 ok
1176064 1176576 ok
1176576 1177088 ok
1177088 1177600 ok
1177600 1178112 ok
1178112 1178624 ok
1178624 1179136 ok
1179136 1179648 ok
1179648 1180160 ok
1180160 1180672 ok
1180672 1181184 ok
1181184 1181696 ok
1181696 1182208 ok
1182208 1182720 ok
1182720 1183232 ok
1183232 1183744 ok
1183744 1184256 ok
1184256 1184768 ok
1184768 1185280 ok
1185280 1185792 ok
1185792 1186304 ok
1186304 1186816 ok
1186816 1187328 ok
1187328 1187840 ok
1187840 1188352 ok
1188352 1188864 ok
1188864 1189376 ok
1189376 1189888 ok
1189888 1190400 ok
1190400 1190912 ok
1190912 1191424 ok
1191424 1191936 ok
1191936 1192448 ok
1192448 1192960 ok
1192960 1193472 ok
1193472 1193984 ok
1193984 1194496 ok
1194496 1195008 ok
1195008 1195520 ok
1195520 1196032 ok
1196032 1196544 ok
1196544 1197056 ok
1197056 1197568 ok
1197568 1198080 ok
1198080 1198592 ok
1198592 1199104 ok
1199104 1199616 ok
1199616 1200128 ok
1200128 1200640 ok
1200640 1201152 ok
1201152 1201664 ok
1201664 1202176 ok
1202176 1202688 ok
1202688 1203200 ok
1203200 1203712 ok
1203712 1204224 ok
1204224 1204736 ok
1204736 1205248 ok
1205248 1205760 ok
1205760 1206272 ok
1206272 1206784 ok
1206784 1207296 ok
1207296 1207808 ok
1207808 1208320 ok
1208320 1208832 ok
1208832 1209344 ok
1209344 1209856 ok
1209856 1210368 ok
1210368 1210880 ok
1210880 1211392 ok
1211392 1211904 ok
1211904 1212416 ok
1212416 1212928 ok
1212928 1213440 ok
1213440 1213952 ok
1213952 1214464 ok
1214464 1214976 ok
1214976 1215488 ok
1215488 1216000 ok
1216000 1216512 ok
1216512 1217024 ok
1217024 1217536 ok
1217536 1218048 ok
1218048 1218560 ok
1218560 1219072 ok
1219072 1219584 ok
1219584 1220096 ok
1220096 1220608 ok
1220608 1221120 ok
1221120 1221632 ok
1221632 1222144 ok
1222144 1222656 ok
1222656 1223168 ok
1223168 1223680 ok
1223680 1224192 ok
1224192 1224704 ok
1224704 1225216 ok
1225216 1225728 ok
1225728 1226240 ok
1226240 1226752 ok
1226752 1227264 ok
1227264 1227776 ok
1227776 1228288 ok
1228288 1228800 ok
1228800 1229312 ok
1229312 1229824 ok
1229824 1230336 ok
1230336 1230848 ok
1230848 1231360 ok
1231360 1231872 ok
1231872 1232384 ok
1232384 1232896 ok
1232896 1233408 ok
1233408 1233920 ok
1233920 1234432 ok
1234432 1234944 ok
1234944 1235456 ok
1235456 1235968 ok
1235968 1236480 ok
1236480 1236992 ok
1236992 1237504 ok
1237504 1238016 ok
1238016 1238528 ok
1238528 1239040 ok
1239040 1239552 ok
1239552 1240064 ok
1240064 1240576 ok
1240576 1241088 ok
1241088 1241600 ok
1241600 1242112 ok
1242112 1242624 ok
1242624 1243136 ok
1243136 1243648 ok
1243648 1244160 ok
1244160 1244672 ok
1244672 1245184 ok
1245184 1245696 ok
1245696 1246208 ok
1246208 1246720 ok
1246720 1247232 ok
1247232 1247744 ok
1247744 1248256 ok
1248256 1248768 ok
1248768 1249280 ok
1249280 1249792 ok
1249792 1250304 ok
1250304 1250816 ok
1250816 1251328 ok
1251328 1251840 ok
1251840 1252352 ok
1252352 1252864 ok
1252864 1253376 ok
1253376 1253888 ok
1253888 1254400 ok
1254400 1254912 ok
1254912 1255424 ok
1255424 1255936 ok
1255936 1256448 ok
1256448 1256960 ok
1256960 1257472 ok
1257472 1257984 ok
1257984 1258496 ok
1258496 1259008 ok
1259008 1259520 ok
1259520 1260032 ok
1260032 1260544 ok
1260544 1261056 ok
1261056 1261568 ok
1261568 1262080 ok
1262080 1262592 ok
1262592 1263104 ok
1263104 1263616 ok
1263616 1264128 ok
1264128 1264640 ok
1264640 1265152 ok
1265152 1265664 ok
1265664 1266176 ok
1266176 1266688 ok
1266688 1267200 ok
1267200 1267712 ok
1267712 1268224 ok
1268224 1268736 ok
1268736 1269248 ok
1269248 1269760 ok
1269760 1270272 ok
1270272 1270784 ok
1270784 1271296 ok
1271296 1271808 ok
1271808 1272320 ok
1272320 1272832 ok
1272832 1273344 ok
1273344 1273856 ok
1273856 1274368 ok
1274368 1274880 ok
1274880 1275392 ok
1275392 1275904 ok
1275904 1276416 ok
1276416 1276928 ok
1276928 1277440 ok
1277440 1277952 ok
1277952 1278464 ok
1278464 1278976 ok
1278976 1279488 ok
1279488 1280000 ok
1280000 1280512 ok
1280512 1281024 ok
1281024 1281536 ok
1281536 1282048 ok
1282048 1282560 ok
1282560 1283072 ok
1283072 1283584 ok
1283584 1284096 ok
1284096 1284608 ok
1284608 1285120 ok
1285120 1285632 ok
1285632 1286144 ok
1286144 1286656 ok
1286656 1287168 ok
1287168 1287680 ok
1287680 1288192 ok
1288192 1288704 ok
1288704 1289216 ok
1289216 1289728 ok
1289728 1290240 ok
1290240 1290752 ok
1290752 1291264 ok
1291264 1291776 ok
1291776 1292288 ok
1292288 1292800 ok
1292800 1293312 ok
1293312 1293824 ok
1293824 1294336 ok
1294336 1294848 ok
1294848 1295360 ok
1295360 1295872 ok
1295872 1296384 ok
1296384 1296896 ok
1296896 1297408 ok
1297408 1297920 ok
1297920 1298432 ok
1298432 1298944 ok
1298944 1299456 ok
1299456 1299968 ok
1299968 1300480 ok
1300480 1300992 ok
1300992 1301504 ok
1301504 1302016 ok
1302016 1302528 ok
1302528 1303040 ok
1303040 1303552 ok
1303552 1304064 ok
1304064 1304576 ok
1304576 1305088 ok
1305088 1305600 ok
1305600 1306112 ok
1306112 1306624 ok
1306624 1307136 ok
1307136 1307648 ok
1307648 1308160 ok
1308160 1308672 ok
1308672 1309184 ok
1309184 1309696 ok
1309696 1310208 ok
1310208 1310720 ok
1310720 1311232 ok
1311232 1311744 ok
1311744 1312256 ok
1312256 1312768 ok
1312768 1313280 ok
1313280 1313792 ok
1313792 1314304 ok
1314304 1314816 ok
1314816 1315328 ok
1315328 1315840 ok
1315840 1316352 ok
1316352 1316864 ok
1316864 1317376 ok
1317376 1317888 ok
1317888 1318400 ok
1318400 1318912 ok
1318912 1319424 ok
1319424 1319936 ok
1319936 1320448 ok
1320448 1320960 ok
1320960 1321472 ok
1321472 1321984 ok
1321984 1322496 ok
1322496 1323008 ok
1323008 1323520 ok
1323520 1324032 ok
1324032 1324544 ok
1324544 1325056 ok
1325056 1325568 ok
1325568 1326080 ok
1326080 1326592 ok
1326592 1327104 ok
1327104 1327616 ok
1327616 1328128 ok
1328128 1328640 ok
1328640 1329152 ok
1329152 1329664 ok
1329664 1330176 ok
1330176 1330688 ok
1330688 1331200 ok
1331200 1331712 ok
1331712 1332224 ok
1332224 1332736 ok
1332736 1333248 ok
1333248 1333760 ok
1333760 1334272 ok
1334272 1334784 ok
1334784 1335296 ok
1335296 1335808 ok
1335808 1336320 ok
1336320 1336832 ok
1336832 1337344 ok
1337344 1337856 ok
1337856 1338368 ok
1338368 1338880 ok
1338880 1339392 ok
1339392 1339904 ok
1339904 1340416 ok
1340416 1340928 ok
1340928 1341440 ok
1341440 1341952 ok
1341952 1342464 ok
1342464 1342976 ok
1342976 1343488 ok
1343488 1344000 ok
1344000 1344512 ok
1344512 1345024 ok
1345024 1345536 ok
1345536 1346048 ok
1346048 1346560 ok
1346560 1347072 ok
1347072 1347584 ok
1347584 1348096 ok
1348096 1348608 ok
1348608 1349120 ok
1349120 1349632 ok
1349632 1350144 ok
1350144 1350656 ok
1350656 1351168 ok
1351168 1351680 ok
1351680 1352192 ok
1352192 1352704 ok
1352704 1353216 ok
1353216 1353728 ok
1353728 1354240 ok
1354240 1354752 ok
1354752 1355264 ok
1355264 1355776 ok
1355776 1356288 ok
1356288 1356800 ok
1356800 1357312 ok
1357312 1357824 ok
1357824 1358336 ok
1358336 1358848 ok
1358848 1359360 ok
1359360 1359872 ok
1359872 1360384 ok
1360384 1360896 ok
1360896 1361408 ok
1361408 1361920 ok
1361920 1362432 ok
1362432 1362944 ok
1362944 1363456 ok
1363456 1363968 ok
1363968 1364480 ok
1364480 1364992 ok
1364992 1365504 ok
1365504 1366016 ok
1366016 1366528 ok
1366528 1367040 ok
1367040 1367552 ok
1367552 1368064 ok
1368064 1368576 ok
1368576 1369088 ok
1369088 1369600 ok
1369600 1370112 ok
1370112 1370624 ok
1370624 1371136 ok
1371136 1371648 ok
1371648 1372160 ok
1372160 1372672 ok
1372672 1373184 ok
1373184 1373696 ok
1373696 1374208 ok
1374208 1374720 ok
1374720 1375232 ok
1375232 1375744 ok
1375744 1376256 ok
1376256 1376768 ok
1376768 1377280 ok
1377280 1377792 ok
1377792 1378304 ok
1378304 1378816 ok
1378816 1379328 ok
1379328 1379840 ok
1379840 1380352 ok
1380352 1380864 ok
1380864 1381376 ok
1381376 1381888 ok
1381888 1382400 ok
1382400 1382912 ok
1382912 1383424 ok
1383424 1383936 ok
1383936 1384448 ok
1384448 1384960 ok
1384960 1385472 ok
1385472 1385984 ok
1385984 1386496 ok
1386496 1387008 ok
1387008 1387520 ok
1387520 1388032 ok
1388032 1388544 ok
1388544 1389056 ok
1389056 1389568 ok
1389568 1390080 ok
1390080 1390592 ok
1390592 1391104 ok
1391104 1391616 ok
1391616 1392128 ok
1392128 1392640 ok
1392640 1393152 ok
1393152 1393664 ok
1393664 1394176 ok
1394176 1394688 ok
1394688 1395200 ok
1395200 1395712 ok
1395712 1396224 ok
1396224 1396736 ok
1396736 1397248 ok
1397248 1397760 ok
1397760 1398272 ok
1398272 1398784 ok
1398784 1399296 ok
1399296 1399808 ok
1399808 1400320 ok
1400320 1400832 ok
1400832 1401344 ok
1401344 1401856 ok
1401856 1402368 ok
1402368 1402880 ok
1402880 1403392 ok
1403392 1403904 ok
1403904 1404416 ok
1404416 1404928 ok
1404928 1405440 ok
1405440 1405952 ok
1405952 1406464 ok
1406464 1406976 ok
1406976 1407488 ok
1407488 1408000 ok
1408000 1408512 ok
1408512 1409024 ok
1409024 1409536 ok
1409536 1410048 ok
1410048 1410560 ok
1410560 1411072 ok
1411072 1411584 ok
1411584 1412096 ok
1412096 1412608 ok
1412608 1413120 ok
1413120 1413632 ok
1413632 1414144 ok
1414144 1414656 ok
1414656 1415168 ok
1415168 1415680 ok
1415680 1416192 ok
1416192 1416704 ok
1416704 1417216 ok
1417216 1417728 ok
1417728 1418240 ok
1418240 1418752 ok
1418752 1419264 ok
1419264 1419776 ok
1419776 1420288 ok
1420288 1420800 ok
1420800 1421312 ok
1421312 1421824 ok
1421824 1422336 ok
1422336 1422848 ok
1422848 1423360 ok
1423360 1423872 ok
1423872 1424384 ok
1424384 1424896 ok
1424896 1425408 ok
1425408 1425920 ok
1425920 1426432 ok
1426432 1426944 ok
1426944 1427456 ok
1427456 1427968 ok
1427968 1428480 ok
1428480 1428992 ok
1428992 1429504 ok
1429504 1430016 ok
1430016 1430528 ok
1430528 1431040 ok
1431040 1431552 ok
1431552 1432064 ok
1432064 1432576 ok
1432576 1433088 ok
1433088 1433600 ok
1433600 1434112 ok
1434112 1434624 ok
1434624 1435136 ok
1435136 1435648 ok
1435648 1436160 ok
1436160 1436672 ok
1436672 1437184 ok
1437184 1437696 ok
1437696 1438208 ok
1438208 1438720 ok
1438720 1439232 ok
1439232 1439744 ok
1439744 1440256 ok
1440256 1440768 ok
1440768 1441280 ok
1441280 1441792 ok
1441792 1442304 ok
1442304 1442816 ok
1442816 1443328 ok
1443328 1443840 ok
1443840 1444352 ok
1444352 1444864 ok
1444864 1445376 ok
1445376 1445888 ok
1445888 1446400 ok
1446400 1446912 ok
1446912 1447424 ok
1447424 1447936 ok
1447936 1448448 ok
1448448 1448960 ok
1448960 1449472 ok
1449472 1449984 ok
1449984 1450496 ok
1450496 1451008 ok
1451008 1451520 ok
1451520 1452032 ok
1452032 1452544 ok
1452544 1453056 ok
1453056 1453568 ok
1453568 1454080 ok
1454080 1454592 ok
1454592 1455104 ok
1455104 1455616 ok
1455616 1456128 ok
1456128 1456640 ok
1456640 1457152 ok
1457152 1457664 ok
1457664 1458176 ok
1458176 1458688 ok
1458688 1459200 ok
1459200 1459712 ok
1459712 1460224 ok
1460224 1460736 ok
1460736 1461248 ok
1461248 1461760 ok
1461760 1462272 ok
1462272 1462784 ok
1462784 1463296 ok
1463296 1463808 ok
1463808 1464320 ok
1464320 1464832 ok
1464832 1465344 ok
1465344 1465856 ok
1465856 1466368 ok
1466368 1466880 ok
1466880 1467392 ok
1467392 1467904 ok
1467904 1468416 ok
1468416 1468928 ok
1468928 1469440 ok
1469440 1469952 ok
1469952 1470464 ok
1470464 1470976 ok
1470976 1471488 ok
1471488 1472000 ok
1472000 1472512 ok
1472512 1473024 ok
1473024 1473536 ok
1473536 1474048 ok
1474048 1474560 ok
1474560 1475072 ok
1475072 1475584 ok
1475584 1476096 ok
1476096 1476608 ok
1476608 1477120 ok
1477120 1477632 ok
1477632 1478144 ok
1478144 1478656 ok
1478656 1479168 ok
1479168 1479680 ok
1479680 1480192 ok
1480192 1480704 ok
1480704 1481216 ok
1481216 1481728 ok
1481728 1482240 ok
1482240 1482752 ok
1482752 1483264 ok
1483264 1483776 ok
1483776 1484288 ok
1484288 1484800 ok
1484800 1485312 ok
1485312 1485824 ok
1485824 1486336 ok
1486336 1486848 ok
1486848 1487360 ok
1487360 1487872 ok
1487872 1488384 ok
1488384 1488896 ok
1488896 1489408 ok
1489408 1489920 ok
1489920 1490432 ok
1490432 1490944 ok
1490944 1491456 ok
1491456 1491968 ok
1491968 1492480 ok
1492480 1492992 ok
1492992 1493504 ok
1493504 1494016 ok
1494016 1494528 ok
1494528 1495040 ok
1495040 1495552 ok
1495552 1496064 ok
1496064 1496576 ok
1496576 1497088 ok
1497088 1497600 ok
1497600 1498112 ok
1498112 1498624 ok
1498624 1499136 ok
1499136 1499648 ok
1499648 1500160 ok
1500160 1500672 ok
1500672 1501184 ok
1501184 1501696 ok
1501696 1502208 ok
1502208 1502720 ok
1502720 1503232 ok
1503232 1503744 ok
1503744 1504256 ok
1504256 1504768 ok
1504768 1505280 ok
1505280 1505792 ok
1505792 1506304 ok
1506304 1506816 ok
1506816 1507328 ok
1507328 1507840 ok
1507840 1508352 ok
1508352 1508864 ok
1508864 1509376 ok
1509376 1509888 ok
1509888 1510400 ok
1510400 1510912 ok
1510912 1511424 ok
1511424 1511936 ok
1511936 1512448 ok
1512448 1512960 ok
1512960 1513472 ok
1513472 1513984 ok
1513984 1514496 ok
1514496 1515008 ok
1515008 1515520 ok
1515520 1516032 ok
1516032 1516544 ok
1516544 1517056 ok
1517056 1517568 ok
1517568 1518080 ok
1518080 1518592 ok
1518592 1519104 ok
1519104 1519616 ok
1519616 1520128 ok
1520128 1520640 ok
1520640 1521152 ok
1521152 1521664 ok
1521664 1522176 ok
1522176 1522688 ok
1522688 1523200 ok
1523200 1523712 ok
1523712 1524224 ok
1524224 1524736 ok
1524736 1525248 ok
1525248 1525760 ok
1525760 1526272 ok
1526272 1526784 ok
1526784 1527296 ok
1527296 1527808 ok
1527808 1528320 ok
1528320 1528832 ok
1528832 1529344 ok
1529344 1529856 ok
1529856 1530368 ok
1530368 1530880 ok
1530880 1531392 ok
1531392 1531904 ok
1531904 1532416 ok
1532416 1532928 ok
1532928 1533440 ok
1533440 1533952 ok
1533952 1534464 ok
1534464 1534976 ok
1534976 1535488 ok
1535488 1536000 ok
1536000 1536512 ok
1536512 1537024 ok
1537024 1537536 ok
1537536 1538048 ok
1538048 1538560 ok
1538560 1539072 ok
1539072 1539584 ok
1539584 1540096 ok
1540096 1540608 ok
1540608 1541120 ok
1541120 1541632 ok
1541632 1542144 ok
1542144 1542656 ok
1542656 1543168 ok
1543168 1543680 ok
1543680 1544192 ok
1544192 1544704 ok
1544704 1545216 ok
1545216 1545728 ok
1545728 1546240 ok
1546240 1546752 ok
1546752 1547264 ok
1547264 1547776 ok
1547776 1548288 ok
1548288 1548800 ok
1548800 1549312 ok
1549312 1549824 ok
1549824 1550336 ok
1550336 1550848 ok
1550848 1551360 ok
1551360 1551872 ok
1551872 1552384 ok
1552384 1552896 ok
1552896 1553408 ok
1553408 1553920 ok
1553920 1554432 ok
1554432 1554944 ok
1554944 1555456 ok
1555456 1555968 ok
1555968 1556480 ok
1556480 1556992 ok
1556992 1557504 ok
1557504 1558016 ok
1558016 1558528 ok
1558528 1559040 ok
1559040 1559552 ok
1559552 1560064 ok
1560064 1560576 ok
1560576 1561088 ok
1561088 1561600 ok
1561600 1562112 ok
1562112 1562624 ok
1562624 1563136 ok
1563136 1563648 ok
1563648 1564160 ok
1564160 1564672 ok
1564672 1565184 ok
1565184 1565696 ok
1565696 1566208 ok
1566208 1566720 ok
1566720 1567232 ok
1567232 1567744 ok
1567744 1568256 ok
1568256 1568768 ok
1568768 1569280 ok
1569280 1569792 ok
1569792 1570304 ok
1570304 1570816 ok
1570816 1571328 ok
1571328 1571840 ok
1571840 1572352 ok
1572352 1572864 ok
1572864 1573376 ok
1573376 1573888 ok
1573888 1574400 ok
1574400 1574912 ok
1574912 1575424 ok
1575424 1575936 ok
1575936 1576448 ok
1576448 1576960 ok
1576960 1577472 ok
1577472 1577984 ok
1577984 1578496 ok
1578496 1579008 ok
1579008 1579520 ok
1579520 1580032 ok
1580032 1580544 ok
1580544 1581056 ok
1581056 1581568 ok
1581568 1582080 ok
1582080 1582592 ok
1582592 1583104 ok
1583104 1583616 ok
1583616 1584128 ok
1584128 1584640 ok
1584640 1585152 ok
1585152 1585664 ok
1585664 1586176 ok
1586176 1586688 ok
1586688 1587200 ok
1587200 1587712 ok
1587712 1588224 ok
1588224 1588736 ok
1588736 1589248 ok
1589248 1589760 ok
1589760 1590272 ok
1590272 1590784 ok
1590784 1591296 ok
1591296 1591808 ok
1591808 1592320 ok
1592320 1592832 ok
1592832 1593344 ok
1593344 1593856 ok
1593856 1594368 ok
1594368 1594880 ok
1594880 1595392 ok
1595392 1595904 ok
1595904 1596416 ok
1596416 1596928 ok
1596928 1597440 ok
1597440 1597952 ok
1597952 1598464 ok
1598464 1598976 ok
1598976 1599488 ok
1599488 1600000 ok
1600000 1600512 ok
1600512 1601024 ok
1601024 1601536 ok
1601536 1602048 ok
1602048 1602560 ok
1602560 1603072 ok
1603072 1603584 ok
1603584 1604096 ok
1604096 1604608 ok
1604608 1605120 ok
1605120 1605632 ok
1605632 1606144 ok
1606144 1606656 ok
1606656 1607168 ok
1607168 1607680 ok
1607680 1608192 ok
1608192 1608704 ok
1608704 1609216 ok
1609216 1609728 ok
1609728 1610240 ok
1610240 1610752 ok
1610752 1611264 ok
1611264 1611776 ok
1611776 1612288 ok
1612288 1612800 ok
1612800 1613312 ok
1613312 1613824 ok
1613824 1614336 ok
1614336 1614848 ok
1614848 1615360 ok
1615360 1615872 ok
1615872 1616384 ok
1616384 1616896 ok
1616896 1617408 ok
1617408 1617920 ok
1617920 1618432 ok
1618432 1618944 ok
1618944 1619456 ok
1619456 1619968 ok
1619968 1620480 ok
1620480 1620992 ok
1620992 1621504 ok
1621504 1622016 ok
1622016 1622528 ok
1622528 1623040 ok
1623040 1623552 ok
1623552 1624064 ok
1624064 1624576 ok
1624576 1625088 ok
1625088 1625600 ok
1625600 1626112 ok
1626112 1626624 ok
1626624 1627136 ok
1627136 1627648 ok
1627648 1628160 ok
1628160 1628672 ok
1628672 1629184 ok
1629184 1629696 ok
1629696 1630208 ok
1630208 1630720 ok
1630720 1631232 ok
1631232 1631744 ok
1631744 1632256 ok
1632256 1632768 ok
1632768 1633280 ok
1633280 1633792 ok
1633792 1634304 ok
1634304 1634816 ok
1634816 1635328 ok
1635328 1635840 ok
1635840 1636352 ok
1636352 1636864 ok
1636864 1637376 ok
1637376 1637888 ok
1637888 1638400 ok
1638400 1638912 ok
1638912 1639424 ok
1639424 1639936 ok
1639936 1640448 ok
1640448 1640960 ok
1640960 1641472 ok
1641472 1641984 ok
1641984 1642496 ok
1642496 1643008 ok
1643008 1643520 ok
1643520 1644032 ok
1644032 1644544 ok
1644544 1645056 ok
1645056 1645568 ok
1645568 1646080 ok
1646080 1646592 ok
1646592 1647104 ok
1647104 1647616 ok
1647616 1648128 ok
1648128 1648640 ok
1648640 1649152 ok
1649152 1649664 ok
1649664 1650176 ok
1650176 1650688 ok
1650688 1651200 ok
1651200 1651712 ok
1651712 1652224 ok
1652224 1652736 ok
1652736 1653248 ok
1653248 1653760 ok
1653760 1654272 ok
1654272 1654784 ok
1654784 1655296 ok
1655296 1655808 ok
1655808 1656320 ok
1656320 1656832 ok
1656832 1657344 ok
1657344 1657856 ok
1657856 1658368 ok
1658368 1658880 ok
1658880 1659392 ok
1659392 1659904 ok
1659904 1660416 ok
1660416 1660928 ok
1660928 1661440 ok
1661440 1661952 ok
1661952 1662464 ok
1662464 1662976 ok
1662976 1663488 ok
1663488 1664000 ok
1664000 1664512 ok
1664512 1665024 ok
1665024 1665536 ok
1665536 1666048 ok
1666048 1666560 ok
1666560 1667072 ok
1667072 1667584 ok
1667584 1668096 ok
1668096 1668608 ok
1668608 1669120 ok
1669120 1669632 ok
1669632 1670144 ok
1670144 1670656 ok
1670656 1671168 ok
1671168 1671680 ok
1671680 1672192 ok
1672192 1672704 ok
1672704 1673216 ok
1673216 1673728 ok
1673728 1674240 ok
1674240 1674752 ok
1674752 1675264 ok
1675264 1675776 ok
1675776 1676288 ok
1676288 1676800 ok
1676800 1677312 ok
1677312 1677824 ok
1677824 1678336 ok
1678336 1678848 ok
1678848 1679360 ok
1679360 1679872 ok
1679872 1680384 ok
1680384 1680896 ok
1680896 1681408 ok
1681408 1681920 ok
1681920 1682432 ok
1682432 1682944 ok
1682944 1683456 ok
1683456 1683968 ok
1683968 1684480 ok
1684480 1684992 ok
1684992 1685504 ok
1685504 1686016 ok
1686016 1686528 ok
1686528 1687040 ok
1687040 1687552 ok
1687552 1688064 ok
1688064 1688576 ok
1688576 1689088 ok
1689088 1689600 ok
1689600 1690112 ok
1690112 1690624 ok
1690624 1691136 ok
1691136 1691648 ok
1691648 1692160 ok
1692160 1692672 ok
1692672 1693184 ok
1693184 1693696 ok
1693696 1694208 ok
1694208 1694720 ok
1694720 1695232 ok
1695232 1695744 ok
1695744 1696256 ok
1696256 1696768 ok
1696768 1697280 ok
1697280 1697792 ok
1697792 1698304 ok
1698304 1698816 ok
1698816 1699328 ok
1699328 1699840 ok
1699840 1700352 ok
1700352 1700864 ok
1700864 1701376 ok
1701376 1701888 ok
1701888 1702400 ok
1702400 1702912 ok
1702912 1703424 ok
1703424 1703936 ok
1703936 1704448 ok
1704448 1704960 ok
1704960 1705472 ok
1705472 1705984 ok
1705984 1706496 ok
1706496 1707008 ok
1707008 1707520 ok
1707520 1708032 ok
1708032 1708544 ok
1708544 1709056 ok
1709056 1709568 ok
1709568 1710080 ok
1710080 1710592 ok
1710592 1711104 ok
1711104 1711616 ok
1711616 1712128 ok
1712128 1712640 ok
1712640 1713152 ok
1713152 1713664 ok
1713664 1714176 ok
1714176 1714688 ok
1714688 1715200 ok
1715200 1715712 ok
1715712 1716224 ok
1716224 1716736 ok
1716736 1717248 ok
1717248 1717760 ok
1717760 1718272 ok
1718272 1718784 ok
1718784 1719296 ok
1719296 1719808 ok
1719808 1720320 ok
1720320 1720832 ok
1720832 1721344 ok
1721344 1721856 ok
1721856 1722368 ok
1722368 1722880 ok
1722880 1723392 ok
1723392 1723904 ok
1723904 1724416 ok
1724416 1724928 ok
1724928 1725440 ok
1725440 1725952 ok
1725952 1726464 ok
1726464 1726976 ok
1726976 1727488 ok
1727488 1728000 ok
1728000 1728512 ok
1728512 1729024 ok
1729024 1729536 ok
1729536 1730048 ok
1730048 1730560 ok
1730560 1731072 ok
1731072 1731584 ok
1731584 1732096 ok
1732096 1732608 ok
1732608 1733120 ok
1733120 1733632 ok
1733632 1734144 ok
1734144 1734656 ok
1734656 1735168 ok
1735168 1735680 ok
1735680 1736192 ok
1736192 1736704 ok
1736704 1737216 ok
1737216 1737728 ok
1737728 1738240 ok
1738240 1738752 ok
1738752 1739264 ok
1739264 1739776 ok
1739776 1740288 ok
1740288 1740800 ok
1740800 1741312 ok
1741312 1741824 ok
1741824 1742336 ok
1742336 1742848 ok
1742848 1743360 ok
1743360 1743872 ok
1743872 1744384 ok
1744384 1744896 ok
1744896 1745408 ok
1745408 1745920 ok
1745920 1746432 ok
1746432 1746944 ok
1746944 1747456 ok
1747456 1747968 ok
1747968 1748480 ok
1748480 1748992 ok
1748992 1749504 ok
1749504 1750016 ok
1750016 1750528 ok
1750528 1751040 ok
1751040 1751552 ok
1751552 1752064 ok
1752064 1752576 ok
1752576 1753088 ok
1753088 1753600 ok
1753600 1754112 ok
1754112 1754624 ok
1754624 1755136 ok
1755136 1755648 ok
1755648 1756160 ok
1756160 1756672 ok
1756672 1757184 ok
1757184 1757696 ok
1757696 1758208 ok
1758208 1758720 ok
1758720 1759232 ok
1759232 1759744 ok
1759744 1760256 ok
1760256 1760768 ok
1760768 1761280 ok
1761280 1761792 ok
1761792 1762304 ok
1762304 1762816 ok
1762816 1763328 ok
1763328 1763840 ok
1763840 1764352 ok
1764352 1764864 ok
1764864 1765376 ok
1765376 1765888 ok
1765888 1766400 ok
1766400 1766912 ok
1766912 1767424 ok
1767424 1767936 ok
1767936 1768448 ok
1768448 1768960 ok
1768960 1769472 ok
1769472 1769984 ok
1769984 1770496 ok
1770496 1771008 ok
1771008 1771520 ok
1771520 1772032 ok
1772032 1772544 ok
1772544 1773056 ok
1773056 1773568 ok
1773568 1774080 ok
1774080 1774592 ok
1774592 1775104 ok
1775104 1775616 ok
1775616 1776128 ok
1776128 1776640 ok
1776640 1777152 ok
1777152 1777664 ok
1777664 1778176 ok
1778176 1778688 ok
1778688 1779200 ok
1779200 1779712 ok
1779712 1780224 ok
1780224 1780736 ok
1780736 1781248 ok
1781248 1781760 ok
1781760 1782272 ok
1782272 1782784 ok
1782784 1783296 ok
1783296 1783808 ok
1783808 1784320 ok
1784320 1784832 ok
1784832 1785344 ok
1785344 1785856 ok
1785856 1786368 ok
1786368 1786880 ok
1786880 1787392 ok
1787392 1787904 ok
1787904 1788416 ok
1788416 1788928 ok
1788928 1789440 ok
1789440 1789952 ok
1789952 1790464 ok
1790464 1790976 ok
1790976 1791488 ok
1791488 1792000 ok
1792000 1792512 ok
1792512 1793024 ok
1793024 1793536 ok
1793536 1794048 ok
1794048 1794560 ok
1794560 1795072 ok
1795072 1795584 ok
1795584 1796096 ok
1796096 1796608 ok
1796608 1797120 ok
1797120 1797632 ok
1797632 1798144 ok
1798144 1798656 ok
1798656 1799168 ok
1799168 1799680 ok
1799680 1800192 ok
1800192 1800704 ok
1800704 1801216 ok
1801216 1801728 ok
1801728 1802240 ok
1802240 1802752 ok
1802752 1803264 ok
1803264 1803776 ok
1803776 1804288 ok
1804288 1804800 ok
1804800 1805312 ok
1805312 1805824 ok
1805824 1806336 ok
1806336 1806848 ok
1806848 1807360 ok
1807360 1807872 ok
1807872 1808384 ok
1808384 1808896 ok
1808896 1809408 ok
1809408 1809920 ok
1809920 1810432 ok
1810432 1810944 ok
1810944 1811456 ok
1811456 1811968 ok
1811968 1812480 ok
1812480 1812992 ok
1812992 1813504 ok
1813504 1814016 ok
1814016 1814528 ok
1814528 1815040 ok
1815040 1815552 ok
1815552 1816064 ok
1816064 1816576 ok
1816576 1817088 ok
1817088 1817600 ok
1817600 1818112 ok
1818112 1818624 ok
1818624 1819136 ok
1819136 1819648 ok
1819648 1820160 ok
1820160 1820672 ok
1820672 1821184 ok
1821184 1821696 ok
1821696 1822208 ok
1822208 1822720 ok
1822720 1823232 ok
1823232 1823744 ok
1823744 1824256 ok
1824256 1824768 ok
1824768 1825280 ok
1825280 1825792 ok
1825792 1826304 ok
1826304 1826816 ok
1826816 1827328 ok
1827328 1827840 ok
1827840 1828352 ok
1828352 1828864 ok
1828864 1829376 ok
1829376 1829888 ok
1829888 1830400 ok
1830400 1830912 ok
1830912 1831424 ok
1831424 1831936 ok
1831936 1832448 ok
1832448 1832960 ok
1832960 1833472 ok
1833472 1833984 ok
1833984 1834496 ok
1834496 1835008 ok
1835008 1835520 ok
1835520 1836032 ok
1836032 1836544 ok
1836544 1837056 ok
1837056 1837568 ok
1837568 1838080 ok
1838080 1838592 ok
1838592 1839104 ok
1839104 1839616 ok
1839616 1840128 ok
1840128 1840640 ok
1840640 1841152 ok
1841152 1841664 ok
1841664 1842176 ok
1842176 1842688 ok
1842688 1843200 ok
1843200 1843712 ok
1843712 1844224 ok
1844224 1844736 ok
1844736 1845248 ok
1845248 1845760 ok
1845760 1846272 ok
1846272 1846784 ok
1846784 1847296 ok
1847296 1847808 ok
1847808 1848320 ok
1848320 1848832 ok
1848832 1849344 ok
1849344 1849856 ok
1849856 1850368 ok
1850368 1850880 ok
1850880 1851392 ok
1851392 1851904 ok
1851904 1852416 ok
1852416 1852928 ok
1852928 1853440 ok
1853440 1853952 ok
1853952 1854464 ok
1854464 1854976 ok
1854976 1855488 ok
1855488 1856000 ok
1856000 1856512 ok
1856512 1857024 ok
1857024 1857536 ok
1857536 1858048 ok
1858048 1858560 ok
1858560 1859072 ok
1859072 1859584 ok
1859584 1860096 ok
1860096 1860608 ok
1860608 1861120 ok
1861120 1861632 ok
1861632 1862144 ok
1862144 1862656 ok
1862656 1863168 ok
1863168 1863680 ok
1863680 1864192 ok
1864192 1864704 ok
1864704 1865216 ok
1865216 1865728 ok
1865728 1866240 ok
1866240 1866752 ok
1866752 1867264 ok
1867264 1867776 ok
1867776 1868288 ok
1868288 1868800 ok
1868800 1869312 ok
1869312 1869824 ok
1869824 1870336 ok
1870336 1870848 ok
1870848 1871360 ok
1871360 1871872 ok
1871872 1872384 ok
1872384 1872896 ok
1872896 1873408 ok
1873408 1873920 ok
1873920 1874432 ok
1874432 1874944 ok
1874944 1875456 ok
1875456 1875968 ok
1875968 1876480 ok
1876480 1876992 ok
1876992 1877504 ok
1877504 1878016 ok
1878016 1878528 ok
1878528 1879040 ok
1879040 1879552 ok
1879552 1880064 ok
1880064 1880576 ok
1880576 1881088 ok
1881088 1881600 ok
1881600 1882112 ok
1882112 1882624 ok
1882624 1883136 ok
1883136 1883648 ok
1883648 1884160 ok
1884160 1884672 ok
1884672 1885184 ok
1885184 1885696 ok
1885696 1886208 ok
1886208 1886720 ok
1886720 1887232 ok
1887232 1887744 ok
1887744 1888256 ok
1888256 1888768 ok
1888768 1889280 ok
1889280 1889792 ok
1889792 1890304 ok
1890304 1890816 ok
1890816 1891328 ok
1891328 1891840 ok
1891840 1892352 ok
1892352 1892864 ok
1892864 1893376 ok
1893376 1893888 ok
1893888 1894400 ok
1894400 1894912 ok
1894912 1895424 ok
1895424 1895936 ok
1895936 1896448 ok
1896448 1896960 ok
1896960 1897472 ok
1897472 1897984 ok
1897984 1898496 ok
1898496 1899008 ok
1899008 1899520 ok
1899520 1900032 ok
1900032 1900544 ok
1900544 1901056 ok
1901056 1901568 ok
1901568 1902080 ok
1902080 1902592 ok
1902592 1903104 ok
1903104 1903616 ok
1903616 1904128 ok
1904128 1904640 ok
1904640 1905152 ok
1905152 1905664 ok
1905664 1906176 ok
1906176 1906688 ok
1906688 1907200 ok
1907200 1907712 ok
1907712 1908224 ok
1908224 1908736 ok
1908736 1909248 ok
1909248 1909760 ok
1909760 1910272 ok
1910272 1910784 ok
1910784 1911296 ok
1911296 1911808 ok
1911808 1912320 ok
1912320 1912832 ok
1912832 1913344 ok
1913344 1913856 ok
1913856 1914368 ok
1914368 1914880 ok
1914880 1915392 ok
1915392 1915904 ok
1915904 1916416 ok
1916416 1916928 ok
1916928 1917440 ok
1917440 1917952 ok
1917952 1918464 ok
1918464 1918976 ok
1918976 1919488 ok
1919488 1920000 ok
1920000 1920512 ok
1920512 1921024 ok
1921024 1921536 ok
1921536 1922048 ok
1922048 1922560 ok
1922560 1923072 ok
1923072 1923584 ok
1923584 1924096 ok
1924096 1924608 ok
1924608 1925120 ok
1925120 1925632 ok
1925632 1926144 ok
1926144 1926656 ok
1926656 1927168 ok
1927168 1927680 ok
1927680 1928192 ok
1928192 1928704 ok
1928704 1929216 ok
1929216 1929728 ok
1929728 1930240 ok
1930240 1930752 ok
1930752 1931264 ok
1931264 1931776 ok
1931776 1932288 ok
1932288 1932800 ok
1932800 1933312 ok
1933312 1933824 ok
1933824 1934336 ok
1934336 1934848 ok
1934848 1935360 ok
1935360 1935872 ok
1935872 1936384 ok
1936384 1936896 ok
1936896 1937408 ok
1937408 1937920 ok
1937920 1938432 ok
1938432 1938944 ok
1938944 1939456 ok
1939456 1939968 ok
1939968 1940480 ok
1940480 1940992 ok
1940992 1941504 ok
1941504 1942016 ok
1942016 1942528 ok
1942528 1943040 ok
1943040 1943552 ok
1943552 1944064 ok
1944064 1944576 ok
1944576 1945088 ok
1945088 1945600 ok
1945600 1946112 ok
1946112 1946624 ok
1946624 1947136 ok
1947136 1947648 ok
1947648 1948160 ok
1948160 1948672 ok
1948672 1949184 ok
1949184 1949696 ok
1949696 1950208 ok
1950208 1950720 ok
1950720 1951232 ok
1951232 1951744 ok
1951744 1952256 ok
1952256 1952768 ok
1952768 1953280 ok
1953280 1953792 ok
1953792 1954304 ok
1954304 1954816 ok
1954816 1955328 ok
1955328 1955840 ok
1955840 1956352 ok
1956352 1956864 ok
1956864 1957376 ok
1957376 1957888 ok
1957888 1958400 ok
1958400 1958912 ok
1958912 1959424 ok
1959424 1959936 ok
1959936 1960448 ok
1960448 1960960 ok
1960960 1961472 ok
1961472 1961984 ok
1961984 1962496 ok
1962496 1963008 ok
1963008 1963520 ok
1963520 1964032 ok
1964032 1964544 ok
1964544 1965056 ok
1965056 1965568 ok
1965568 1966080 ok
Sep 22 23:21:47.660 DEBG IO Write 1023 has deps [JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:47.660 DEBG up_ds_listen was notified
Sep 22 23:21:47.660 DEBG up_ds_listen process 1023
Sep 22 23:21:47.660 DEBG [A] ack job 1023:24, : downstairs
Sep 22 23:21:47.660 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::test_pantry_import_from_url_ovmf ... ok
{"msg":"Scrub at offset 256/3840 sp:256","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.705402357Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Scrub at offset 512/3840 sp:512","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:47.929518055Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:47.993 DEBG IO Write 1024 has deps [JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:47.993 DEBG up_ds_listen was notified
Sep 22 23:21:47.993 DEBG up_ds_listen process 1024
Sep 22 23:21:47.993 DEBG [A] ack job 1024:25, : downstairs
Sep 22 23:21:47.993 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:47.994 DEBG IO Flush 1025 has deps [JobId(1024), JobId(1023), JobId(1022)]
Sep 22 23:21:48.060 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:2
Sep 22 23:21:48.061 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:2
Sep 22 23:21:48.061 DEBG Flush     :1002 extent_limit None deps:[JobId(1001), JobId(1000)] res:true f:1 g:2
{"msg":"Scrub at offset 768/3840 sp:768","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:48.151829863Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:48.324 DEBG IO Write 1026 has deps [JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:48.324 DEBG up_ds_listen was notified
Sep 22 23:21:48.324 DEBG up_ds_listen process 1026
Sep 22 23:21:48.325 DEBG [A] ack job 1026:27, : downstairs
Sep 22 23:21:48.325 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"Scrub at offset 1024/3840 sp:1024","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:48.381645289Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:48.391 INFO [0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected, looper: 0
Sep 22 23:21:48.391 INFO [0] Proc runs for 127.0.0.1:63038 in state Disconnected
Sep 22 23:21:48.391 INFO [1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected, looper: 1
Sep 22 23:21:48.391 INFO [1] Proc runs for 127.0.0.1:50898 in state Disconnected
Sep 22 23:21:48.391 INFO [2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected, looper: 2
Sep 22 23:21:48.391 INFO [2] Proc runs for 127.0.0.1:62519 in state Disconnected
Sep 22 23:21:48.391 INFO accepted connection from 127.0.0.1:51418, task: main
Sep 22 23:21:48.391 INFO accepted connection from 127.0.0.1:53793, task: main
Sep 22 23:21:48.391 INFO accepted connection from 127.0.0.1:59016, task: main
Sep 22 23:21:48.392 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:48.392 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:48.392 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:48.392 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:48.392 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:48.392 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 14046fa9-5850-443d-9708-a7d5d0130e56, gen: 1 } connected, version 4, task: proc
Sep 22 23:21:48.392 INFO [0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) Disconnected Disconnected Disconnected ds_transition to WaitActive
Sep 22 23:21:48.392 INFO [0] Transition from Disconnected to WaitActive
Sep 22 23:21:48.392 INFO [1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) WaitActive Disconnected Disconnected ds_transition to WaitActive
Sep 22 23:21:48.392 INFO [1] Transition from Disconnected to WaitActive
Sep 22 23:21:48.392 INFO [2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (14046fa9-5850-443d-9708-a7d5d0130e56) WaitActive WaitActive Disconnected ds_transition to WaitActive
Sep 22 23:21:48.392 INFO [2] Transition from Disconnected to WaitActive
{"msg":"Checking if live repair is needed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:48.41778269Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"No Live Repair required at this time","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:48.417819229Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:48.421483099Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/9890f602-aa95-4bcc-8172-78f624d84944/is_finished","method":"GET","req_id":"fa0c3c86-5e5c-47eb-8776-4c9243b7e4df","remote_addr":"127.0.0.1:33149","local_addr":"127.0.0.1:49824","component":"dropshot","latency_us":221,"response_code":"200"}
Sep 22 23:21:48.610 DEBG Flush     :1006 extent_limit None deps:[JobId(1005), JobId(1004), JobId(1003)] res:true f:2 g:2
Sep 22 23:21:48.610 DEBG Flush     :1006 extent_limit None deps:[JobId(1005), JobId(1004), JobId(1003)] res:true f:2 g:2
Sep 22 23:21:48.610 DEBG Flush     :1006 extent_limit None deps:[JobId(1005), JobId(1004), JobId(1003)] res:true f:2 g:2
{"msg":"Scrub at offset 1280/3840 sp:1280","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:48.611409232Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:48.720 DEBG IO Write 1027 has deps [JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:48.721 DEBG up_ds_listen was notified
Sep 22 23:21:48.721 DEBG up_ds_listen process 1027
Sep 22 23:21:48.721 DEBG [A] ack job 1027:28, : downstairs
Sep 22 23:21:48.721 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"Scrub at offset 1536/3840 sp:1536","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:48.835603452Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:48.867 DEBG IO Flush 1028 has deps [JobId(1027), JobId(1026), JobId(1025)]
Sep 22 23:21:48.898 DEBG Write     :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:48.929 DEBG Write     :1002 deps:[JobId(1001)] res:true
Sep 22 23:21:48.931 WARN returning error on write!
Sep 22 23:21:48.931 DEBG Write     :1002 deps:[JobId(1001)] res:false
Sep 22 23:21:48.932 WARN returning error on write!
Sep 22 23:21:48.932 DEBG Write     :1002 deps:[JobId(1001)] res:false
Sep 22 23:21:48.933 WARN returning error on write!
Sep 22 23:21:48.933 DEBG Write     :1002 deps:[JobId(1001)] res:false
Sep 22 23:21:48.963 DEBG Write     :1002 deps:[JobId(1001)] res:true
{"msg":"Scrub at offset 1792/3840 sp:1792","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:49.058909813Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:49.078 DEBG Flush     :1009 extent_limit None deps:[JobId(1008), JobId(1007)] res:true f:3 g:2
Sep 22 23:21:49.078 DEBG Flush     :1009 extent_limit None deps:[JobId(1008), JobId(1007)] res:true f:3 g:2
Sep 22 23:21:49.078 DEBG Flush     :1009 extent_limit None deps:[JobId(1008), JobId(1007)] res:true f:3 g:2
{"msg":"Scrub at offset 2048/3840 sp:2048","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:49.280701324Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:49.296 DEBG IO Write 1029 has deps [JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:49.296 DEBG up_ds_listen was notified
Sep 22 23:21:49.296 DEBG up_ds_listen process 1029
Sep 22 23:21:49.296 DEBG [A] ack job 1029:30, : downstairs
Sep 22 23:21:49.296 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:49.478791397Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/9890f602-aa95-4bcc-8172-78f624d84944/is_finished","method":"GET","req_id":"d63d82ca-abbb-4210-9877-afca8aab02a6","remote_addr":"127.0.0.1:33149","local_addr":"127.0.0.1:49824","component":"dropshot","latency_us":302,"response_code":"200"}
{"msg":"Scrub at offset 2304/3840 sp:2304","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:49.509808243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:49.579 DEBG Flush     :1012 extent_limit None deps:[JobId(1011), JobId(1010)] res:true f:4 g:2
Sep 22 23:21:49.579 DEBG Flush     :1012 extent_limit None deps:[JobId(1011), JobId(1010)] res:true f:4 g:2
Sep 22 23:21:49.579 DEBG Flush     :1012 extent_limit None deps:[JobId(1011), JobId(1010)] res:true f:4 g:2
Sep 22 23:21:49.627 DEBG IO Write 1030 has deps [JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:49.627 DEBG up_ds_listen was notified
Sep 22 23:21:49.627 DEBG up_ds_listen process 1030
Sep 22 23:21:49.627 DEBG [A] ack job 1030:31, : downstairs
Sep 22 23:21:49.628 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:49.628 DEBG IO Flush 1031 has deps [JobId(1030), JobId(1029), JobId(1028)]
{"msg":"Scrub at offset 2560/3840 sp:2560","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:49.734112018Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Scrub at offset 2816/3840 sp:2816","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:49.959521184Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:49.960 DEBG IO Write 1032 has deps [JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:49.960 DEBG up_ds_listen was notified
Sep 22 23:21:49.960 DEBG up_ds_listen process 1032
Sep 22 23:21:49.961 DEBG [A] ack job 1032:33, : downstairs
Sep 22 23:21:49.961 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:49.961 INFO [lossy] sleeping 1 second
Sep 22 23:21:49.961 INFO [lossy] sleeping 1 second
Sep 22 23:21:50.081 DEBG Flush     :1015 extent_limit None deps:[JobId(1014), JobId(1013)] res:true f:5 g:2
Sep 22 23:21:50.081 DEBG Flush     :1015 extent_limit None deps:[JobId(1014), JobId(1013)] res:true f:5 g:2
Sep 22 23:21:50.081 DEBG Flush     :1015 extent_limit None deps:[JobId(1014), JobId(1013)] res:true f:5 g:2
{"msg":"Scrub at offset 3072/3840 sp:3072","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:50.184335937Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:50.296 DEBG IO Write 1033 has deps [JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:50.296 DEBG up_ds_listen was notified
Sep 22 23:21:50.296 DEBG up_ds_listen process 1033
Sep 22 23:21:50.296 DEBG [A] ack job 1033:34, : downstairs
Sep 22 23:21:50.296 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:50.297 DEBG IO Flush 1034 has deps [JobId(1033), JobId(1032), JobId(1031)]
{"msg":"Scrub at offset 3328/3840 sp:3328","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:50.408004743Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:50.511401061Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/9890f602-aa95-4bcc-8172-78f624d84944/is_finished","method":"GET","req_id":"c5c4385f-82e6-4f2d-a92a-752103ae423c","remote_addr":"127.0.0.1:33149","local_addr":"127.0.0.1:49824","component":"dropshot","latency_us":226,"response_code":"200"}
Sep 22 23:21:50.630 DEBG IO Write 1035 has deps [JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:50.630 DEBG up_ds_listen was notified
Sep 22 23:21:50.630 DEBG up_ds_listen process 1035
Sep 22 23:21:50.630 DEBG [A] ack job 1035:36, : downstairs
Sep 22 23:21:50.630 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:50.635 DEBG Flush     :1019 extent_limit None deps:[JobId(1018), JobId(1017), JobId(1016)] res:true f:6 g:2
Sep 22 23:21:50.635 DEBG Flush     :1019 extent_limit None deps:[JobId(1018), JobId(1017), JobId(1016)] res:true f:6 g:2
Sep 22 23:21:50.635 DEBG Flush     :1019 extent_limit None deps:[JobId(1018), JobId(1017), JobId(1016)] res:true f:6 g:2
{"msg":"Scrub at offset 3584/3840 sp:3584","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:50.636669624Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Scrub at offset 3840/3840 sp:3840","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:50.861882643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"Scrub 3dd77b9b-1720-4c07-b5d9-e4b24cceb521 done in 3 seconds. Retries:0 scrub_size:131072 size:3840 pause_milli:0","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:50.863047201Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:50.864 DEBG Flush     :1021 extent_limit None deps:[JobId(1020)] res:true f:7 g:2
Sep 22 23:21:50.865 DEBG Flush     :1021 extent_limit None deps:[JobId(1020)] res:true f:7 g:2
Sep 22 23:21:50.865 DEBG Flush     :1021 extent_limit None deps:[JobId(1020)] res:true f:7 g:2
Sep 22 23:21:50.965 DEBG IO Write 1036 has deps [JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:50.965 DEBG up_ds_listen was notified
Sep 22 23:21:50.965 DEBG up_ds_listen process 1036
Sep 22 23:21:50.965 DEBG [A] ack job 1036:37, : downstairs
Sep 22 23:21:50.965 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:50.966 DEBG IO Flush 1037 has deps [JobId(1036), JobId(1035), JobId(1034)]
Sep 22 23:21:51.302 DEBG IO Write 1038 has deps [JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:51.303 DEBG up_ds_listen was notified
Sep 22 23:21:51.303 DEBG up_ds_listen process 1038
Sep 22 23:21:51.303 DEBG [A] ack job 1038:39, : downstairs
Sep 22 23:21:51.303 DEBG up_ds_listen checked 1 jobs, back to waiting
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.513245698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/9890f602-aa95-4bcc-8172-78f624d84944/is_finished","method":"GET","req_id":"63665cfe-9e11-49bd-a25a-bf021ba54baa","remote_addr":"127.0.0.1:33149","local_addr":"127.0.0.1:49824","component":"dropshot","latency_us":261,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.513956333Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/job/9890f602-aa95-4bcc-8172-78f624d84944/ok","method":"GET","req_id":"fe74781b-b47a-41bc-a63b-2ab540a52781","remote_addr":"127.0.0.1:33149","local_addr":"127.0.0.1:49824","component":"dropshot","latency_us":176,"response_code":"200"}
{"msg":"detach removing entry for volume 3dd77b9b-1720-4c07-b5d9-e4b24cceb521","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.514540503Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"detaching volume 3dd77b9b-1720-4c07-b5d9-e4b24cceb521","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.514572568Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
Sep 22 23:21:51.515 DEBG Flush     :1022 extent_limit None deps:[] res:true f:8 g:2
Sep 22 23:21:51.515 DEBG Flush     :1022 extent_limit None deps:[] res:true f:8 g:2
Sep 22 23:21:51.515 DEBG Flush     :1022 extent_limit None deps:[] res:true f:8 g:2
{"msg":"Request to deactivate this guest","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.515742904Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b set deactivating.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.515774597Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.517899118Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"uri":"/crucible/pantry/0/volume/3dd77b9b-1720-4c07-b5d9-e4b24cceb521","method":"DELETE","req_id":"eef1dd5b-b8e1-4310-a7a4-faedfe430dc6","remote_addr":"127.0.0.1:33149","local_addr":"127.0.0.1:49824","component":"dropshot","latency_us":3458,"response_code":"204"}
{"msg":"Upstairs starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.518296855Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible Version: BuildInfo {\n    version: \"0.0.1\",\n    git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n    git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n    git_branch: \"main\",\n    rustc_semver: \"1.70.0\",\n    rustc_channel: \"stable\",\n    rustc_host_triple: \"x86_64-unknown-illumos\",\n    rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n    cargo_triple: \"x86_64-unknown-illumos\",\n    debug: true,\n    opt_level: 0,\n}","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.518346199Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.518369783Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible stats registered with UUID: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.518392301Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Crucible 71f26a6c-b7c6-4f58-901b-ae4e36b1710b has session id: 3a34d657-eced-4b79-a336-c052da87a51b","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.518414628Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] connecting to 127.0.0.1:63038","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.518500951Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
{"msg":"[1] connecting to 127.0.0.1:50898","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.51862315Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1"}
{"msg":"[2] connecting to 127.0.0.1:62519","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.51871462Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2"}
{"msg":"up_listen starts","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.518753921Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"task":"up_listen"}
{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.51877829Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Flush timeout: 0.5","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.518799021Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.518967198Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0"}
{"msg":"[0] Proc runs for 127.0.0.1:63038 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.519003653Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.519026985Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1"}
{"msg":"[1] Proc runs for 127.0.0.1:50898 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.519049632Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b looper connected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.519071218Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2"}
{"msg":"[2] Proc runs for 127.0.0.1:62519 in state New","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.519093319Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:51.519 INFO accepted connection from 127.0.0.1:57723, task: main
Sep 22 23:21:51.519 INFO accepted connection from 127.0.0.1:56683, task: main
Sep 22 23:21:51.519 INFO accepted connection from 127.0.0.1:46008, task: main
Sep 22 23:21:51.519 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:51.519 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:51.519 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:51.519 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } connected, version 4, task: proc
Sep 22 23:21:51.519 INFO Connection request from 71f26a6c-b7c6-4f58-901b-ae4e36b1710b with version 4, task: proc
Sep 22 23:21:51.519 INFO upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } connected, version 4, task: proc
{"msg":"[0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (b703b4b5-d321-4299-acb9-588c806f0017) New New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.519905725Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.51993482Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (b703b4b5-d321-4299-acb9-588c806f0017) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.519958982Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.519980114Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (b703b4b5-d321-4299-acb9-588c806f0017) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.520004041Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.520024927Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
The guest has requested activation
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b active request set","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.520138046Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] received activate with gen 3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.520175072Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] client got ds_active_rx, promote! session b703b4b5-d321-4299-acb9-588c806f0017","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.52020435Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] received activate with gen 3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.520226225Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] client got ds_active_rx, promote! session b703b4b5-d321-4299-acb9-588c806f0017","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.520248115Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] received activate with gen 3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.520268672Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] client got ds_active_rx, promote! session b703b4b5-d321-4299-acb9-588c806f0017","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.520290517Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:51.520 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 }
Sep 22 23:21:51.520 WARN Signaling to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } thread that UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } is being promoted (read-write)
Sep 22 23:21:51.520 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 }
Sep 22 23:21:51.520 WARN Signaling to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } thread that UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } is being promoted (read-write)
Sep 22 23:21:51.520 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 }
Sep 22 23:21:51.520 WARN Signaling to UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 } thread that UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } is being promoted (read-write)
Sep 22 23:21:51.520 WARN Another upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 }, task: main
Sep 22 23:21:51.521 INFO UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } is now active (read-write)
Sep 22 23:21:51.521 WARN Another upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 }, task: main
Sep 22 23:21:51.521 INFO UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } is now active (read-write)
Sep 22 23:21:51.521 WARN Another upstairs UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: 3d6550fa-8c01-4792-b110-6653df927fd3, gen: 2 }, task: main
Sep 22 23:21:51.521 INFO UpstairsConnection { upstairs_id: 71f26a6c-b7c6-4f58-901b-ae4e36b1710b, session_id: b703b4b5-d321-4299-acb9-588c806f0017, gen: 3 } is now active (read-write)
Sep 22 23:21:51.521 INFO connection (127.0.0.1:50361): all done
Sep 22 23:21:51.521 INFO connection (127.0.0.1:56375): all done
Sep 22 23:21:51.521 INFO connection (127.0.0.1:64643): all done
{"msg":"[0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) cmd_loop saw YouAreNoLongerActive 71f26a6c-b7c6-4f58-901b-ae4e36b1710b b703b4b5-d321-4299-acb9-588c806f0017 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:51.521869318Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) Active Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.521902383Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.521928896Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b set inactive, session 3d6550fa-8c01-4792-b110-6653df927fd3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.521952984Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:63038: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:51.521977398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.52200266Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b connection to 127.0.0.1:63038 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522025966Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"0","component":"datafile"}
{"msg":"[1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) cmd_loop saw YouAreNoLongerActive 71f26a6c-b7c6-4f58-901b-ae4e36b1710b b703b4b5-d321-4299-acb9-588c806f0017 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:51.522052875Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) Disconnected Active Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522077657Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522099737Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b set inactive, session 3d6550fa-8c01-4792-b110-6653df927fd3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522123206Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:50898: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:51.522147071Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522187423Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b connection to 127.0.0.1:50898 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522213643Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"1","component":"datafile"}
{"msg":"[2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) cmd_loop saw YouAreNoLongerActive 71f26a6c-b7c6-4f58-901b-ae4e36b1710b b703b4b5-d321-4299-acb9-588c806f0017 3","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:51.522240618Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (3d6550fa-8c01-4792-b110-6653df927fd3) Disconnected Disconnected Active ds_transition to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522265466Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] Transition from Active to Disabled","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522287567Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b set inactive, session 3d6550fa-8c01-4792-b110-6653df927fd3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522310805Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"127.0.0.1:62519: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 3 than ours 2","v":0,"name":"crucible-pantry","level":50,"time":"2023-09-22T23:21:51.522334391Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b Gone missing, transition from Disabled to Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522359243Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b connection to 127.0.0.1:62519 closed","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522382187Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"looper":"2","component":"datafile"}
{"msg":"[0] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:51.522404847Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:63038 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522426834Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522449417Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] 127.0.0.1:63038 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.52247139Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:50898 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522493552Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522516341Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] 127.0.0.1:50898 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522544589Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:62519 task reports connection:false","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522567798Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b Disconnected Disconnected Disconnected","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522590787Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] 127.0.0.1:62519 task reports offline","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522612859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[1] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:51.522634691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[2] pm_task rx.recv() is None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:51.522656398Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769,"component":"datafile"}
{"msg":"[0] downstairs client at 127.0.0.1:63038 has UUID 0c4b6300-aa56-4191-90c9-1d56da8be939","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522679698Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 0c4b6300-aa56-4191-90c9-1d56da8be939, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522705518Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522727248Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] downstairs client at 127.0.0.1:50898 has UUID 77625cc5-cf57-4c89-b7d2-c29dca5216c1","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522749067Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 77625cc5-cf57-4c89-b7d2-c29dca5216c1, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522774744Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522796162Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] downstairs client at 127.0.0.1:62519 has UUID d89769b3-2748-4a20-ad6b-6b02a9292676","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522817945Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: d89769b3-2748-4a20-ad6b-6b02a9292676, encrypted: true, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.52284346Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitActive WaitActive WaitActive","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.522864942Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:51.530 INFO Current flush_numbers [0..12]: [1, 2, 3, 4, 5, 6, 6, 7, 0, 0, 0, 0]
Sep 22 23:21:51.531 INFO Current flush_numbers [0..12]: [1, 2, 3, 4, 5, 6, 6, 7, 0, 0, 0, 0]
Sep 22 23:21:51.532 INFO Current flush_numbers [0..12]: [1, 2, 3, 4, 5, 6, 6, 7, 0, 0, 0, 0]
Sep 22 23:21:51.544 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:51.545 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:21:51.546 INFO Downstairs has completed Negotiation, task: proc
{"msg":"[0] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (b703b4b5-d321-4299-acb9-588c806f0017) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.546539353Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.5465708Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:51.546592826Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.546613678Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (b703b4b5-d321-4299-acb9-588c806f0017) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.546783161Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.546811843Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:51.546833342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.546854151Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 71f26a6c-b7c6-4f58-901b-ae4e36b1710b (b703b4b5-d321-4299-acb9-588c806f0017) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.54701687Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547045538Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible-pantry","level":40,"time":"2023-09-22T23:21:51.547067052Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547087715Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] 127.0.0.1:63038 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547109103Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547130859Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R flush_numbers[0..12]: [1, 2, 3, 4, 5, 6, 6, 7, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547152465Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R generation[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547174165Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547196683Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R flush_numbers[0..12]: [1, 2, 3, 4, 5, 6, 6, 7, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547223167Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R generation[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547245613Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547267999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2]R flush_numbers[0..12]: [1, 2, 3, 4, 5, 6, 6, 7, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547289429Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2]R generation[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547310676Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547332986Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Max found gen is 3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547353342Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Generation requested: 3 >= found:3","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547373819Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Next flush: 8","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547393681Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
The guest has finished waiting for activation
{"msg":"All extents match","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547414006Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"No downstairs repair required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.54743484Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"No initial repair work was required","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.54745545Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547476085Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b is now active with session: b703b4b5-d321-4299-acb9-588c806f0017","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547498451Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b Set Active after no repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547519735Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547541003Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547561349Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] 127.0.0.1:50898 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547582474Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547603604Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547627817Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] 127.0.0.1:62519 task reports connection:true","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547649387Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"71f26a6c-b7c6-4f58-901b-ae4e36b1710b Active Active Active","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547670691Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"Set check for repair","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547690969Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547711487Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547731999Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.54775227Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547772602Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547793138Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547813372Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] received reconcile message","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547834062Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547854455Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible-pantry","level":30,"time":"2023-09-22T23:21:51.547874649Z","hostname":"ip-10-150-1-55.us-west-2.compute.internal","pid":4769}
Sep 22 23:21:51.589 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:51.641 DEBG IO Write 1039 has deps [JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:51.641 DEBG up_ds_listen was notified
Sep 22 23:21:51.641 DEBG up_ds_listen process 1039
Sep 22 23:21:51.642 DEBG [A] ack job 1039:40, : downstairs
Sep 22 23:21:51.642 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:51.643 DEBG IO Flush 1040 has deps [JobId(1039), JobId(1038), JobId(1037)]
Sep 22 23:21:51.647 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:51.704 DEBG Read      :1000 deps:[] res:true
Sep 22 23:21:52.038 DEBG IO Write 1041 has deps [JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:52.038 DEBG up_ds_listen was notified
Sep 22 23:21:52.038 DEBG up_ds_listen process 1041
Sep 22 23:21:52.038 DEBG [A] ack job 1041:42, : downstairs
Sep 22 23:21:52.038 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:52.198 INFO [lossy] sleeping 1 second
Sep 22 23:21:52.198 INFO [lossy] sleeping 1 second
Sep 22 23:21:52.200 WARN returning error on write!
Sep 22 23:21:52.200 DEBG Write     :1003 deps:[JobId(1001)] res:false
Sep 22 23:21:52.201 INFO [lossy] skipping 1003
Sep 22 23:21:52.201 INFO [lossy] skipping 1003
Sep 22 23:21:52.230 DEBG Write     :1003 deps:[JobId(1001)] res:true
Sep 22 23:21:52.231 INFO [lossy] sleeping 1 second
Sep 22 23:21:52.565 DEBG IO Write 1042 has deps [JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:52.565 DEBG IO Flush 1043 has deps [JobId(1042), JobId(1041), JobId(1040)]
Sep 22 23:21:52.565 DEBG up_ds_listen was notified
Sep 22 23:21:52.565 DEBG up_ds_listen process 1042
Sep 22 23:21:52.565 DEBG [A] ack job 1042:43, : downstairs
Sep 22 23:21:52.565 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:52.905 DEBG IO Write 1044 has deps [JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:52.905 DEBG up_ds_listen was notified
Sep 22 23:21:52.905 DEBG up_ds_listen process 1044
Sep 22 23:21:52.905 DEBG [A] ack job 1044:45, : downstairs
Sep 22 23:21:52.905 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:53.246 DEBG IO Write 1045 has deps [JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:53.246 DEBG up_ds_listen was notified
Sep 22 23:21:53.246 DEBG up_ds_listen process 1045
Sep 22 23:21:53.247 DEBG [A] ack job 1045:46, : downstairs
Sep 22 23:21:53.247 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:53.247 DEBG IO Flush 1046 has deps [JobId(1045), JobId(1044), JobId(1043)]
Sep 22 23:21:53.247 INFO [lossy] skipping 1003
Sep 22 23:21:53.247 INFO [lossy] skipping 1003
Sep 22 23:21:53.279 DEBG Write     :1003 deps:[JobId(1001)] res:true
Sep 22 23:21:53.280 INFO [lossy] sleeping 1 second
Sep 22 23:21:53.280 INFO [lossy] skipping 1003
Sep 22 23:21:53.280 INFO [lossy] skipping 1004
Sep 22 23:21:53.280 WARN returning error on write!
Sep 22 23:21:53.280 DEBG Write     :1003 deps:[JobId(1001)] res:false
Sep 22 23:21:53.281 INFO [lossy] skipping 1004
Sep 22 23:21:53.310 DEBG Write     :1003 deps:[JobId(1001)] res:true
Sep 22 23:21:53.324 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001)] res:true f:2 g:1
Sep 22 23:21:53.324 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001)] res:true f:2 g:1
Sep 22 23:21:53.653 DEBG IO Write 1047 has deps [JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:53.653 DEBG up_ds_listen was notified
Sep 22 23:21:53.653 DEBG up_ds_listen process 1047
Sep 22 23:21:53.653 DEBG [A] ack job 1047:48, : downstairs
Sep 22 23:21:53.653 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:53.983 DEBG IO Write 1048 has deps [JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:53.983 DEBG up_ds_listen was notified
Sep 22 23:21:53.983 DEBG up_ds_listen process 1048
Sep 22 23:21:53.983 DEBG [A] ack job 1048:49, : downstairs
Sep 22 23:21:53.983 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:53.984 DEBG IO Flush 1049 has deps [JobId(1048), JobId(1047), JobId(1046)]
test test::test_pantry_scrub ... ok
Sep 22 23:21:54.317 DEBG IO Write 1050 has deps [JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:54.317 DEBG up_ds_listen was notified
Sep 22 23:21:54.317 DEBG up_ds_listen process 1050
Sep 22 23:21:54.317 DEBG [A] ack job 1050:51, : downstairs
Sep 22 23:21:54.317 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:54.324 DEBG Flush     :1004 extent_limit None deps:[JobId(1003), JobId(1002), JobId(1001)] res:true f:2 g:1
Sep 22 23:21:54.654 DEBG IO Write 1051 has deps [JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:54.654 DEBG up_ds_listen was notified
Sep 22 23:21:54.654 DEBG up_ds_listen process 1051
Sep 22 23:21:54.654 DEBG [A] ack job 1051:52, : downstairs
Sep 22 23:21:54.654 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:54.655 DEBG IO Flush 1052 has deps [JobId(1051), JobId(1050), JobId(1049)]
Sep 22 23:21:54.986 DEBG IO Write 1053 has deps [JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:54.986 DEBG up_ds_listen was notified
Sep 22 23:21:54.986 DEBG up_ds_listen process 1053
Sep 22 23:21:54.986 DEBG [A] ack job 1053:54, : downstairs
Sep 22 23:21:54.986 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:55.320 DEBG IO Write 1054 has deps [JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:55.320 DEBG up_ds_listen was notified
Sep 22 23:21:55.320 DEBG up_ds_listen process 1054
Sep 22 23:21:55.321 DEBG [A] ack job 1054:55, : downstairs
Sep 22 23:21:55.321 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:55.321 DEBG IO Flush 1055 has deps [JobId(1054), JobId(1053), JobId(1052)]
Sep 22 23:21:55.717 DEBG IO Write 1056 has deps [JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:55.717 DEBG up_ds_listen was notified
Sep 22 23:21:55.717 DEBG up_ds_listen process 1056
Sep 22 23:21:55.717 DEBG [A] ack job 1056:57, : downstairs
Sep 22 23:21:55.717 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:55.866 INFO [lossy] sleeping 1 second
Sep 22 23:21:55.866 INFO [lossy] skipping 1005
Sep 22 23:21:55.866 INFO [lossy] skipping 1005
Sep 22 23:21:55.868 WARN returning error on write!
Sep 22 23:21:55.868 DEBG Write     :1005 deps:[JobId(1004), JobId(1001)] res:false
Sep 22 23:21:55.869 WARN returning error on write!
Sep 22 23:21:55.869 DEBG Write     :1005 deps:[JobId(1004), JobId(1001)] res:false
Sep 22 23:21:55.870 INFO [lossy] skipping 1005
Sep 22 23:21:55.899 DEBG Write     :1005 deps:[JobId(1004), JobId(1001)] res:true
Sep 22 23:21:55.929 DEBG Write     :1005 deps:[JobId(1004), JobId(1001)] res:true
Sep 22 23:21:56.259 DEBG IO Write 1057 has deps [JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:56.260 DEBG IO Flush 1058 has deps [JobId(1057), JobId(1056), JobId(1055)]
Sep 22 23:21:56.260 DEBG up_ds_listen was notified
Sep 22 23:21:56.260 DEBG up_ds_listen process 1057
Sep 22 23:21:56.260 DEBG [A] ack job 1057:58, : downstairs
Sep 22 23:21:56.260 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:56.593 DEBG IO Write 1059 has deps [JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:56.593 DEBG up_ds_listen was notified
Sep 22 23:21:56.593 DEBG up_ds_listen process 1059
Sep 22 23:21:56.593 DEBG [A] ack job 1059:60, : downstairs
Sep 22 23:21:56.594 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:56.932 DEBG IO Write 1060 has deps [JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:56.932 DEBG up_ds_listen was notified
Sep 22 23:21:56.932 DEBG up_ds_listen process 1060
Sep 22 23:21:56.932 DEBG [A] ack job 1060:61, : downstairs
Sep 22 23:21:56.932 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:56.933 DEBG IO Flush 1061 has deps [JobId(1060), JobId(1059), JobId(1058)]
Sep 22 23:21:56.933 INFO [lossy] skipping 1005
Sep 22 23:21:56.933 INFO [lossy] skipping 1005
Sep 22 23:21:56.964 DEBG Write     :1005 deps:[JobId(1004), JobId(1001)] res:true
Sep 22 23:21:57.297 DEBG IO Write 1062 has deps [JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:57.298 DEBG up_ds_listen was notified
Sep 22 23:21:57.298 DEBG up_ds_listen process 1062
Sep 22 23:21:57.298 DEBG [A] ack job 1062:63, : downstairs
Sep 22 23:21:57.298 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:57.636 DEBG IO Write 1063 has deps [JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:57.636 DEBG up_ds_listen was notified
Sep 22 23:21:57.636 DEBG up_ds_listen process 1063
Sep 22 23:21:57.636 DEBG [A] ack job 1063:64, : downstairs
Sep 22 23:21:57.636 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:57.637 DEBG IO Flush 1064 has deps [JobId(1063), JobId(1062), JobId(1061)]
Sep 22 23:21:57.974 DEBG IO Write 1065 has deps [JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:57.974 DEBG up_ds_listen was notified
Sep 22 23:21:57.974 DEBG up_ds_listen process 1065
Sep 22 23:21:57.974 DEBG [A] ack job 1065:66, : downstairs
Sep 22 23:21:57.974 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:58.115 DEBG IO Write 1000 has deps []
Sep 22 23:21:58.115 INFO Checking if live repair is needed
Sep 22 23:21:58.115 INFO No Live Repair required at this time
Sep 22 23:21:58.115 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:21:58.115 DEBG up_ds_listen was notified
Sep 22 23:21:58.115 DEBG up_ds_listen process 1000
Sep 22 23:21:58.118 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:21:58.118 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:58.314 DEBG IO Write 1066 has deps [JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:58.315 DEBG up_ds_listen was notified
Sep 22 23:21:58.315 DEBG up_ds_listen process 1066
Sep 22 23:21:58.315 DEBG [A] ack job 1066:67, : downstairs
Sep 22 23:21:58.315 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:58.315 DEBG IO Flush 1067 has deps [JobId(1066), JobId(1065), JobId(1064)]
Sep 22 23:21:58.656 DEBG IO Write 1068 has deps [JobId(1067), JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:58.656 DEBG up_ds_listen was notified
Sep 22 23:21:58.656 DEBG up_ds_listen process 1068
Sep 22 23:21:58.656 DEBG [A] ack job 1068:69, : downstairs
Sep 22 23:21:58.657 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:59.059 DEBG IO Write 1069 has deps [JobId(1067), JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)]
Sep 22 23:21:59.059 DEBG up_ds_listen was notified
Sep 22 23:21:59.060 DEBG up_ds_listen process 1069
Sep 22 23:21:59.060 DEBG [A] ack job 1069:70, : downstairs
Sep 22 23:21:59.060 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:21:59.214 DEBG IO Flush 1070 has deps [JobId(1069), JobId(1068), JobId(1067)]
Sep 22 23:21:59.214 INFO [lossy] skipping 1006
Sep 22 23:21:59.246 DEBG Write     :1006 deps:[JobId(1004), JobId(1001)] res:true
Sep 22 23:21:59.247 WARN returning error on flush!
Sep 22 23:21:59.247 DEBG Flush     :1007 extent_limit None deps:[JobId(1006), JobId(1005), JobId(1004)] res:false f:3 g:1
Sep 22 23:21:59.278 DEBG Write     :1006 deps:[JobId(1004), JobId(1001)] res:true
Sep 22 23:21:59.279 INFO [lossy] skipping 1006
Sep 22 23:21:59.279 INFO [lossy] skipping 1007
Sep 22 23:21:59.279 INFO [lossy] skipping 1006
Sep 22 23:21:59.280 WARN returning error on write!
Sep 22 23:21:59.280 DEBG Write     :1006 deps:[JobId(1004), JobId(1001)] res:false
Sep 22 23:21:59.280 WARN returning error on write!
Sep 22 23:21:59.280 DEBG Write     :1006 deps:[JobId(1004), JobId(1001)] res:false
Sep 22 23:21:59.281 WARN returning error on write!
Sep 22 23:21:59.281 DEBG Write     :1006 deps:[JobId(1004), JobId(1001)] res:false
Sep 22 23:21:59.312 DEBG Write     :1006 deps:[JobId(1004), JobId(1001)] res:true
Sep 22 23:21:59.332 DEBG Flush     :1007 extent_limit None deps:[JobId(1006), JobId(1005), JobId(1004)] res:true f:3 g:1
Sep 22 23:21:59.332 DEBG Flush     :1007 extent_limit None deps:[JobId(1006), JobId(1005), JobId(1004)] res:true f:3 g:1
Sep 22 23:21:59.332 DEBG Flush     :1007 extent_limit None deps:[JobId(1006), JobId(1005), JobId(1004)] res:true f:3 g:1
Sep 22 23:21:59.333 DEBG IO Read  1071 has deps [JobId(1070)]
Sep 22 23:21:59.538 INFO [lossy] sleeping 1 second
Sep 22 23:21:59.569 DEBG Write     :1008 deps:[JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:21:59.570 INFO [lossy] sleeping 1 second
Sep 22 23:21:59.814 DEBG Write     :1009 deps:[JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:21:59.822 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007)] res:true f:4 g:1
Sep 22 23:21:59.822 DEBG IO Flush 1072 has deps [JobId(1071), JobId(1070)]
Sep 22 23:22:00.038 INFO [lossy] sleeping 1 second
Sep 22 23:22:00.107 ERRO [0] job id 1000 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [0] job id 1000 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [0] job id 1007 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [1] job id 1003 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [1] job id 1005 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [1] job id 1005 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [2] job id 1000 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [2] job id 1000 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [2] job id 1002 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [2] job id 1002 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [2] job id 1002 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [2] job id 1003 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [2] job id 1006 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [2] job id 1006 saw error GenericError("test error")
Sep 22 23:22:00.107 ERRO [2] job id 1006 saw error GenericError("test error")
Sep 22 23:22:00.108 DEBG up_ds_listen was notified
Sep 22 23:22:00.108 DEBG up_ds_listen process 1001
Sep 22 23:22:00.108 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:22:00.109 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:22:00.109 DEBG up_ds_listen process 1004
Sep 22 23:22:00.109 DEBG [A] ack job 1004:5, : downstairs
Sep 22 23:22:00.112 DEBG [rc] retire 1004 clears [JobId(1002), JobId(1003), JobId(1004)], : downstairs
Sep 22 23:22:00.112 DEBG up_ds_listen process 1007
Sep 22 23:22:00.113 DEBG [A] ack job 1007:8, : downstairs
Sep 22 23:22:00.116 DEBG [rc] retire 1007 clears [JobId(1005), JobId(1006), JobId(1007)], : downstairs
Sep 22 23:22:00.116 DEBG up_ds_listen checked 3 jobs, back to waiting
Sep 22 23:22:00.116 DEBG up_ds_listen was notified
Sep 22 23:22:00.116 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:22:00.116 DEBG up_ds_listen was notified
Sep 22 23:22:00.116 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:22:00.730 DEBG Write     :1008 deps:[JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:00.761 DEBG Write     :1009 deps:[JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:00.769 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007)] res:true f:4 g:1
Sep 22 23:22:00.770 WARN returning error on write!
Sep 22 23:22:00.770 DEBG Write     :1011 deps:[JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:00.801 DEBG Write     :1012 deps:[JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:00.803 INFO [lossy] skipping 1014
Sep 22 23:22:00.833 DEBG Write     :1011 deps:[JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:00.842 DEBG Flush     :1013 extent_limit None deps:[JobId(1012), JobId(1011), JobId(1010)] res:true f:5 g:1
Sep 22 23:22:00.873 DEBG Write     :1014 deps:[JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:00.875 INFO [lossy] sleeping 1 second
Sep 22 23:22:00.876 WARN returning error on write!
Sep 22 23:22:00.876 DEBG Write     :1008 deps:[JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:00.878 WARN returning error on write!
Sep 22 23:22:00.878 DEBG Write     :1009 deps:[JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:00.878 INFO [lossy] skipping 1012
Sep 22 23:22:00.878 INFO [lossy] skipping 1013
Sep 22 23:22:00.908 DEBG Write     :1008 deps:[JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:00.941 DEBG Write     :1009 deps:[JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:00.949 DEBG Flush     :1010 extent_limit None deps:[JobId(1009), JobId(1008), JobId(1007)] res:true f:4 g:1
Sep 22 23:22:00.980 DEBG Write     :1011 deps:[JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:00.981 INFO [lossy] skipping 1012
Sep 22 23:22:00.981 WARN 1013 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:00.981 WARN 1014 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:00.981 WARN 1015 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:00.981 INFO [lossy] skipping 1012
Sep 22 23:22:00.981 INFO [lossy] skipping 1012
Sep 22 23:22:00.981 INFO [lossy] skipping 1012
Sep 22 23:22:01.013 DEBG Write     :1012 deps:[JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:01.014 INFO [lossy] sleeping 1 second
Sep 22 23:22:01.223 WARN returning error on write!
Sep 22 23:22:01.223 DEBG Write     :1011 deps:[JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:01.223 INFO [lossy] skipping 1012
Sep 22 23:22:01.223 INFO [lossy] skipping 1016
Sep 22 23:22:01.224 WARN returning error on write!
Sep 22 23:22:01.224 DEBG Write     :1011 deps:[JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:01.256 DEBG Write     :1012 deps:[JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:01.258 WARN returning error on write!
Sep 22 23:22:01.258 DEBG Write     :1011 deps:[JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:01.288 DEBG Write     :1011 deps:[JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:01.296 DEBG Flush     :1013 extent_limit None deps:[JobId(1012), JobId(1011), JobId(1010)] res:true f:5 g:1
Sep 22 23:22:01.327 DEBG Write     :1014 deps:[JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:01.360 DEBG Write     :1015 deps:[JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:01.368 DEBG Flush     :1016 extent_limit None deps:[JobId(1015), JobId(1014), JobId(1013)] res:true f:6 g:1
Sep 22 23:22:01.368 INFO [lossy] sleeping 1 second
Sep 22 23:22:01.453 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:22:01.453 INFO Created new region file "/tmp/downstairs-zrMnlo6G/region.json"
Sep 22 23:22:01.760 INFO current number of open files limit 65536 is already the maximum
Sep 22 23:22:01.760 INFO Opened existing region file "/tmp/downstairs-zrMnlo6G/region.json"
Sep 22 23:22:01.760 INFO Database read version 1
Sep 22 23:22:01.760 INFO Database write version 1
Sep 22 23:22:01.812 INFO UUID: c3c61ac0-c12a-42ef-b3a2-a316e079f741
Sep 22 23:22:01.812 INFO Blocks per extent:512 Total Extents: 188
Sep 22 23:22:01.812 INFO Crucible Version: Crucible Version: 0.0.1
Commit SHA: ed48f294784d46ea7d4bb99336918b74358eca46
Commit timestamp: 2023-09-22T22:51:18.000000000Z  branch: main
rustc: 1.70.0 stable x86_64-unknown-illumos
Cargo: x86_64-unknown-illumos  Debug: true Opt level: 0, task: main
Sep 22 23:22:01.812 INFO Upstairs <-> Downstairs Message Version: 4, task: main
Sep 22 23:22:01.812 INFO Using address: 127.0.0.1:42129, task: main
Sep 22 23:22:01.813 INFO Repair listens on 127.0.0.1:0, task: repair
Sep 22 23:22:01.813 DEBG registered endpoint, path: /extent/{eid}/files, method: GET, local_addr: 127.0.0.1:56184, task: repair
Sep 22 23:22:01.813 DEBG registered endpoint, path: /newextent/{eid}/{file_type}, method: GET, local_addr: 127.0.0.1:56184, task: repair
Sep 22 23:22:01.813 INFO listening, local_addr: 127.0.0.1:56184, task: repair
Sep 22 23:22:01.813 DEBG successfully registered DTrace USDT probes, local_addr: 127.0.0.1:56184, task: repair
Sep 22 23:22:01.813 INFO Using repair address: 127.0.0.1:56184, task: main
Sep 22 23:22:01.813 INFO No SSL acceptor configured, task: main
Sep 22 23:22:01.821 INFO listening on 127.0.0.1:0, task: main
Sep 22 23:22:01.821 WARN 9144e02c-c312-47c4-9b1c-f03618834608 request to replace downstairs 127.0.0.1:52165 with 127.0.0.1:42129
Sep 22 23:22:01.821 INFO 9144e02c-c312-47c4-9b1c-f03618834608 found old target: 127.0.0.1:52165 at 0
Sep 22 23:22:01.821 INFO 9144e02c-c312-47c4-9b1c-f03618834608 replacing old: 127.0.0.1:52165 at 0
Sep 22 23:22:01.821 INFO [0] client skip 2 in process jobs because fault, : downstairs
Sep 22 23:22:01.821 INFO [0] changed 2 jobs to fault skipped, : downstairs
Sep 22 23:22:01.821 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) Active Active Active ds_transition to Replacing
Sep 22 23:22:01.821 INFO [0] Transition from Active to Replacing
Sep 22 23:22:02.026 DEBG Write     :1015 deps:[JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.034 DEBG Flush     :1016 extent_limit None deps:[JobId(1015), JobId(1014), JobId(1013)] res:true f:6 g:1
Sep 22 23:22:02.035 WARN returning error on write!
Sep 22 23:22:02.035 DEBG Write     :1017 deps:[JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:02.035 INFO [lossy] skipping 1018
Sep 22 23:22:02.035 INFO [lossy] skipping 1021
Sep 22 23:22:02.035 INFO [lossy] skipping 1017
Sep 22 23:22:02.065 DEBG Write     :1018 deps:[JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.066 INFO [lossy] skipping 1021
Sep 22 23:22:02.097 DEBG Write     :1017 deps:[JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.105 DEBG Flush     :1019 extent_limit None deps:[JobId(1018), JobId(1017), JobId(1016)] res:true f:7 g:1
Sep 22 23:22:02.136 DEBG Write     :1020 deps:[JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.137 INFO [lossy] skipping 1021
Sep 22 23:22:02.137 INFO [lossy] skipping 1022
Sep 22 23:22:02.168 DEBG Write     :1021 deps:[JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.176 DEBG Flush     :1022 extent_limit None deps:[JobId(1021), JobId(1020), JobId(1019)] res:true f:8 g:1
Sep 22 23:22:02.176 INFO [lossy] sleeping 1 second
Sep 22 23:22:02.182 DEBG Flush     :1013 extent_limit None deps:[JobId(1012), JobId(1011), JobId(1010)] res:true f:5 g:1
Sep 22 23:22:02.184 WARN returning error on write!
Sep 22 23:22:02.184 DEBG Write     :1014 deps:[JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:02.214 DEBG Write     :1015 deps:[JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.216 WARN 1016 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:02.216 INFO [lossy] skipping 1017
Sep 22 23:22:02.216 INFO [lossy] skipping 1018
Sep 22 23:22:02.216 INFO [lossy] skipping 1021
Sep 22 23:22:02.216 INFO [lossy] skipping 1014
Sep 22 23:22:02.216 INFO [lossy] skipping 1014
Sep 22 23:22:02.246 DEBG Write     :1014 deps:[JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.248 INFO [lossy] skipping 1016
Sep 22 23:22:02.254 DEBG Flush     :1016 extent_limit None deps:[JobId(1015), JobId(1014), JobId(1013)] res:true f:6 g:1
Sep 22 23:22:02.285 DEBG Write     :1017 deps:[JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.287 WARN returning error on write!
Sep 22 23:22:02.287 DEBG Write     :1018 deps:[JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:02.287 WARN 1019 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:02.287 INFO [lossy] skipping 1020
Sep 22 23:22:02.287 INFO [lossy] skipping 1021
Sep 22 23:22:02.287 INFO [lossy] skipping 1018
Sep 22 23:22:02.287 INFO [lossy] skipping 1020
Sep 22 23:22:02.287 INFO [lossy] skipping 1021
Sep 22 23:22:02.287 INFO [lossy] skipping 1018
Sep 22 23:22:02.287 INFO [lossy] skipping 1020
Sep 22 23:22:02.287 WARN 1021 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:02.287 INFO [lossy] skipping 1018
Sep 22 23:22:02.287 WARN 1020 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:02.317 DEBG Write     :1018 deps:[JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.325 DEBG Flush     :1019 extent_limit None deps:[JobId(1018), JobId(1017), JobId(1016)] res:true f:7 g:1
Sep 22 23:22:02.356 DEBG Write     :1020 deps:[JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.388 DEBG Write     :1021 deps:[JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.390 WARN returning error on flush!
Sep 22 23:22:02.390 DEBG Flush     :1022 extent_limit None deps:[JobId(1021), JobId(1020), JobId(1019)] res:false f:8 g:1
Sep 22 23:22:02.396 DEBG Flush     :1022 extent_limit None deps:[JobId(1021), JobId(1020), JobId(1019)] res:true f:8 g:1
Sep 22 23:22:02.396 INFO [lossy] sleeping 1 second
Sep 22 23:22:02.397 WARN returning error on write!
Sep 22 23:22:02.397 DEBG Write     :1017 deps:[JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:02.428 DEBG Write     :1018 deps:[JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.429 INFO [lossy] skipping 1017
Sep 22 23:22:02.459 DEBG Write     :1017 deps:[JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.460 INFO [lossy] skipping 1019
Sep 22 23:22:02.460 INFO [lossy] skipping 1021
Sep 22 23:22:02.460 INFO [lossy] skipping 1022
Sep 22 23:22:02.467 DEBG Flush     :1019 extent_limit None deps:[JobId(1018), JobId(1017), JobId(1016)] res:true f:7 g:1
Sep 22 23:22:02.467 INFO [lossy] skipping 1021
Sep 22 23:22:02.467 WARN 1022 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:02.497 DEBG Write     :1021 deps:[JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.499 INFO [lossy] skipping 1020
Sep 22 23:22:02.499 WARN 1022 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:02.500 WARN returning error on write!
Sep 22 23:22:02.500 DEBG Write     :1020 deps:[JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:02.501 WARN returning error on write!
Sep 22 23:22:02.501 DEBG Write     :1020 deps:[JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:02.501 INFO [lossy] skipping 1020
Sep 22 23:22:02.502 WARN returning error on write!
Sep 22 23:22:02.502 DEBG Write     :1020 deps:[JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:02.532 DEBG Write     :1020 deps:[JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:02.534 WARN returning error on flush!
Sep 22 23:22:02.534 DEBG Flush     :1022 extent_limit None deps:[JobId(1021), JobId(1020), JobId(1019)] res:false f:8 g:1
Sep 22 23:22:02.534 INFO [lossy] skipping 1022
Sep 22 23:22:02.534 INFO [lossy] skipping 1022
Sep 22 23:22:02.534 WARN returning error on flush!
Sep 22 23:22:02.534 DEBG Flush     :1022 extent_limit None deps:[JobId(1021), JobId(1020), JobId(1019)] res:false f:8 g:1
Sep 22 23:22:02.540 DEBG Flush     :1022 extent_limit None deps:[JobId(1021), JobId(1020), JobId(1019)] res:true f:8 g:1
Sep 22 23:22:02.752 INFO [lossy] sleeping 1 second
Sep 22 23:22:03.215 DEBG Write     :1023 deps:[JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.246 DEBG Write     :1024 deps:[JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.247 WARN returning error on flush!
Sep 22 23:22:03.247 DEBG Flush     :1025 extent_limit None deps:[JobId(1024), JobId(1023), JobId(1022)] res:false f:9 g:1
Sep 22 23:22:03.247 INFO [lossy] skipping 1025
Sep 22 23:22:03.253 DEBG Flush     :1025 extent_limit None deps:[JobId(1024), JobId(1023), JobId(1022)] res:true f:9 g:1
Sep 22 23:22:03.283 DEBG Write     :1026 deps:[JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.284 INFO [lossy] sleeping 1 second
Sep 22 23:22:03.530 DEBG Write     :1023 deps:[JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.531 INFO [lossy] skipping 1024
Sep 22 23:22:03.561 DEBG Write     :1024 deps:[JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.569 DEBG Flush     :1025 extent_limit None deps:[JobId(1024), JobId(1023), JobId(1022)] res:true f:9 g:1
Sep 22 23:22:03.569 INFO [lossy] skipping 1026
Sep 22 23:22:03.599 DEBG Write     :1027 deps:[JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.600 WARN 1028 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:03.600 INFO [lossy] skipping 1026
Sep 22 23:22:03.601 WARN returning error on write!
Sep 22 23:22:03.601 DEBG Write     :1026 deps:[JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:03.602 WARN returning error on write!
Sep 22 23:22:03.602 DEBG Write     :1026 deps:[JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:03.602 WARN returning error on write!
Sep 22 23:22:03.602 DEBG Write     :1026 deps:[JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:03.633 DEBG Write     :1026 deps:[JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.640 DEBG Flush     :1028 extent_limit None deps:[JobId(1027), JobId(1026), JobId(1025)] res:true f:10 g:1
Sep 22 23:22:03.640 INFO [lossy] sleeping 1 second
Sep 22 23:22:03.886 DEBG Write     :1023 deps:[JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.888 WARN returning error on write!
Sep 22 23:22:03.888 DEBG Write     :1024 deps:[JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:03.888 INFO [lossy] skipping 1025
Sep 22 23:22:03.888 INFO [lossy] skipping 1026
Sep 22 23:22:03.918 DEBG Write     :1024 deps:[JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.926 DEBG Flush     :1025 extent_limit None deps:[JobId(1024), JobId(1023), JobId(1022)] res:true f:9 g:1
Sep 22 23:22:03.927 WARN returning error on write!
Sep 22 23:22:03.927 DEBG Write     :1026 deps:[JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:03.957 DEBG Write     :1026 deps:[JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.988 DEBG Write     :1027 deps:[JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:03.995 DEBG Flush     :1028 extent_limit None deps:[JobId(1027), JobId(1026), JobId(1025)] res:true f:10 g:1
Sep 22 23:22:03.996 WARN returning error on write!
Sep 22 23:22:03.996 DEBG Write     :1029 deps:[JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:03.997 INFO [lossy] skipping 1029
Sep 22 23:22:03.997 INFO [lossy] skipping 1029
Sep 22 23:22:04.027 DEBG Write     :1029 deps:[JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:04.028 INFO [lossy] sleeping 1 second
Sep 22 23:22:04.456 INFO [lossy] skipping 1027
Sep 22 23:22:04.456 INFO [lossy] skipping 1030
Sep 22 23:22:04.487 DEBG Write     :1027 deps:[JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:04.494 DEBG Flush     :1028 extent_limit None deps:[JobId(1027), JobId(1026), JobId(1025)] res:true f:10 g:1
Sep 22 23:22:04.494 INFO [lossy] skipping 1029
Sep 22 23:22:04.524 DEBG Write     :1030 deps:[JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:04.525 WARN 1031 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:04.526 WARN returning error on write!
Sep 22 23:22:04.526 DEBG Write     :1029 deps:[JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:04.555 DEBG Write     :1029 deps:[JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:04.556 INFO [lossy] skipping 1031
Sep 22 23:22:04.562 DEBG Flush     :1031 extent_limit None deps:[JobId(1030), JobId(1029), JobId(1028)] res:true f:11 g:1
Sep 22 23:22:04.562 INFO [lossy] sleeping 1 second
Sep 22 23:22:04.806 DEBG Write     :1029 deps:[JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:04.837 DEBG Write     :1030 deps:[JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:04.844 DEBG Flush     :1031 extent_limit None deps:[JobId(1030), JobId(1029), JobId(1028)] res:true f:11 g:1
Sep 22 23:22:04.844 INFO [lossy] skipping 1032
Sep 22 23:22:04.874 DEBG Write     :1033 deps:[JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:04.905 DEBG Write     :1032 deps:[JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:04.912 DEBG Flush     :1034 extent_limit None deps:[JobId(1033), JobId(1032), JobId(1031)] res:true f:12 g:1
Sep 22 23:22:04.912 INFO [lossy] sleeping 1 second
Sep 22 23:22:05.156 DEBG Write     :1030 deps:[JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:05.157 WARN returning error on flush!
Sep 22 23:22:05.157 DEBG Flush     :1031 extent_limit None deps:[JobId(1030), JobId(1029), JobId(1028)] res:false f:11 g:1
Sep 22 23:22:05.164 DEBG Flush     :1031 extent_limit None deps:[JobId(1030), JobId(1029), JobId(1028)] res:true f:11 g:1
Sep 22 23:22:05.164 INFO [lossy] skipping 1032
Sep 22 23:22:05.164 INFO [lossy] skipping 1033
Sep 22 23:22:05.164 WARN 1034 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:05.164 INFO [lossy] skipping 1035
Sep 22 23:22:05.164 INFO [lossy] skipping 1032
Sep 22 23:22:05.193 DEBG Write     :1033 deps:[JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:05.194 WARN 1035 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:05.194 INFO [lossy] skipping 1032
Sep 22 23:22:05.224 DEBG Write     :1032 deps:[JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:05.225 INFO [lossy] sleeping 1 second
Sep 22 23:22:05.714 WARN returning error on write!
Sep 22 23:22:05.714 DEBG Write     :1032 deps:[JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:05.715 INFO [lossy] skipping 1033
Sep 22 23:22:05.715 INFO [lossy] skipping 1037
Sep 22 23:22:05.715 INFO [lossy] skipping 1038
Sep 22 23:22:05.715 WARN returning error on write!
Sep 22 23:22:05.715 DEBG Write     :1032 deps:[JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:05.716 INFO [lossy] skipping 1033
Sep 22 23:22:05.716 INFO [lossy] skipping 1037
Sep 22 23:22:05.745 DEBG Write     :1032 deps:[JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:05.746 INFO [lossy] skipping 1033
Sep 22 23:22:05.746 INFO [lossy] skipping 1037
Sep 22 23:22:05.747 WARN returning error on write!
Sep 22 23:22:05.747 DEBG Write     :1033 deps:[JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:05.777 DEBG Write     :1033 deps:[JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:05.778 INFO [lossy] sleeping 1 second
Sep 22 23:22:06.037 DEBG Write     :1035 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:06.038 WARN returning error on write!
Sep 22 23:22:06.038 DEBG Write     :1036 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:06.039 INFO [lossy] skipping 1039
Sep 22 23:22:06.068 DEBG Write     :1036 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:06.069 INFO [lossy] skipping 1039
Sep 22 23:22:06.069 INFO [lossy] skipping 1039
Sep 22 23:22:06.069 INFO [lossy] sleeping 1 second
Sep 22 23:22:06.297 DEBG Flush     :1034 extent_limit None deps:[JobId(1033), JobId(1032), JobId(1031)] res:true f:12 g:1
Sep 22 23:22:06.328 DEBG Write     :1035 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:06.358 DEBG Write     :1036 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:06.365 DEBG Flush     :1037 extent_limit None deps:[JobId(1036), JobId(1035), JobId(1034)] res:true f:13 g:1
Sep 22 23:22:06.395 DEBG Write     :1038 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:06.425 DEBG Write     :1039 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:06.426 INFO [lossy] skipping 1040
Sep 22 23:22:06.432 DEBG Flush     :1040 extent_limit None deps:[JobId(1039), JobId(1038), JobId(1037)] res:true f:14 g:1
Sep 22 23:22:06.433 WARN returning error on write!
Sep 22 23:22:06.433 DEBG Write     :1041 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:06.434 WARN returning error on write!
Sep 22 23:22:06.434 DEBG Write     :1041 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:06.464 DEBG Write     :1041 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:06.465 INFO [lossy] sleeping 1 second
Sep 22 23:22:06.906 INFO [lossy] skipping 1034
Sep 22 23:22:06.906 INFO [lossy] skipping 1036
Sep 22 23:22:06.906 INFO [lossy] skipping 1039
Sep 22 23:22:06.906 INFO [lossy] skipping 1040
Sep 22 23:22:06.906 INFO [lossy] skipping 1043
Sep 22 23:22:06.912 DEBG Flush     :1034 extent_limit None deps:[JobId(1033), JobId(1032), JobId(1031)] res:true f:12 g:1
Sep 22 23:22:06.945 DEBG Write     :1036 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:06.946 INFO [lossy] skipping 1043
Sep 22 23:22:06.976 DEBG Write     :1035 deps:[JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:06.977 WARN returning error on flush!
Sep 22 23:22:06.977 DEBG Flush     :1037 extent_limit None deps:[JobId(1036), JobId(1035), JobId(1034)] res:false f:13 g:1
Sep 22 23:22:06.977 INFO [lossy] skipping 1038
Sep 22 23:22:06.977 WARN 1041 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:06.977 WARN 1042 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:06.977 INFO [lossy] skipping 1043
Sep 22 23:22:06.977 WARN 1044 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:22:06.983 DEBG Flush     :1037 extent_limit None deps:[JobId(1036), JobId(1035), JobId(1034)] res:true f:13 g:1
Sep 22 23:22:07.013 DEBG Write     :1038 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.044 DEBG Write     :1039 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.051 DEBG Flush     :1040 extent_limit None deps:[JobId(1039), JobId(1038), JobId(1037)] res:true f:14 g:1
Sep 22 23:22:07.052 WARN returning error on write!
Sep 22 23:22:07.052 DEBG Write     :1041 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:07.054 WARN returning error on write!
Sep 22 23:22:07.054 DEBG Write     :1042 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:07.055 WARN 1043 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:07.055 WARN 1044 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:07.084 DEBG Write     :1041 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.086 WARN returning error on write!
Sep 22 23:22:07.086 DEBG Write     :1042 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:07.117 DEBG Write     :1042 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.118 WARN returning error on flush!
Sep 22 23:22:07.118 DEBG Flush     :1043 extent_limit None deps:[JobId(1042), JobId(1041), JobId(1040)] res:false f:15 g:1
Sep 22 23:22:07.124 DEBG Flush     :1043 extent_limit None deps:[JobId(1042), JobId(1041), JobId(1040)] res:true f:15 g:1
Sep 22 23:22:07.155 DEBG Write     :1044 deps:[JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.156 INFO [lossy] sleeping 1 second
Sep 22 23:22:07.156 WARN returning error on flush!
Sep 22 23:22:07.156 DEBG Flush     :1037 extent_limit None deps:[JobId(1036), JobId(1035), JobId(1034)] res:false f:13 g:1
Sep 22 23:22:07.157 INFO [lossy] skipping 1041
Sep 22 23:22:07.157 INFO [lossy] skipping 1042
Sep 22 23:22:07.157 INFO [lossy] skipping 1043
Sep 22 23:22:07.163 DEBG Flush     :1037 extent_limit None deps:[JobId(1036), JobId(1035), JobId(1034)] res:true f:13 g:1
Sep 22 23:22:07.194 DEBG Write     :1038 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.226 DEBG Write     :1039 deps:[JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.227 WARN returning error on flush!
Sep 22 23:22:07.227 DEBG Flush     :1040 extent_limit None deps:[JobId(1039), JobId(1038), JobId(1037)] res:false f:14 g:1
Sep 22 23:22:07.227 INFO [lossy] skipping 1043
Sep 22 23:22:07.227 INFO [lossy] skipping 1044
Sep 22 23:22:07.234 DEBG Flush     :1040 extent_limit None deps:[JobId(1039), JobId(1038), JobId(1037)] res:true f:14 g:1
Sep 22 23:22:07.234 INFO [lossy] skipping 1043
Sep 22 23:22:07.234 INFO [lossy] skipping 1044
Sep 22 23:22:07.234 INFO [lossy] skipping 1043
Sep 22 23:22:07.234 WARN 1044 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:07.234 WARN 1043 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:07.265 DEBG Write     :1041 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.267 WARN returning error on write!
Sep 22 23:22:07.267 DEBG Write     :1042 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:07.267 WARN 1043 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:07.267 INFO [lossy] skipping 1042
Sep 22 23:22:07.268 WARN returning error on write!
Sep 22 23:22:07.268 DEBG Write     :1042 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:07.269 WARN returning error on write!
Sep 22 23:22:07.269 DEBG Write     :1042 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:07.269 INFO [lossy] skipping 1042
Sep 22 23:22:07.300 DEBG Write     :1042 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.308 DEBG Flush     :1043 extent_limit None deps:[JobId(1042), JobId(1041), JobId(1040)] res:true f:15 g:1
Sep 22 23:22:07.309 WARN returning error on write!
Sep 22 23:22:07.309 DEBG Write     :1044 deps:[JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:07.340 DEBG Write     :1044 deps:[JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.341 INFO [lossy] sleeping 1 second
Sep 22 23:22:07.577 INFO [lossy] skipping 1042
Sep 22 23:22:07.577 INFO [lossy] skipping 1044
Sep 22 23:22:07.578 WARN returning error on write!
Sep 22 23:22:07.578 DEBG Write     :1042 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:07.608 DEBG Write     :1042 deps:[JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:07.609 INFO [lossy] sleeping 1 second
Sep 22 23:22:08.347 WARN returning error on write!
Sep 22 23:22:08.347 DEBG Write     :1045 deps:[JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:08.377 DEBG Write     :1045 deps:[JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:08.385 DEBG Flush     :1046 extent_limit None deps:[JobId(1045), JobId(1044), JobId(1043)] res:true f:16 g:1
Sep 22 23:22:08.386 WARN returning error on write!
Sep 22 23:22:08.386 DEBG Write     :1047 deps:[JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:08.386 INFO [lossy] skipping 1048
Sep 22 23:22:08.386 WARN 1049 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:08.417 DEBG Write     :1047 deps:[JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:08.449 DEBG Write     :1048 deps:[JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:08.450 INFO [lossy] skipping 1049
Sep 22 23:22:08.456 DEBG Flush     :1049 extent_limit None deps:[JobId(1048), JobId(1047), JobId(1046)] res:true f:17 g:1
Sep 22 23:22:08.457 INFO [lossy] sleeping 1 second
Sep 22 23:22:08.458 WARN returning error on write!
Sep 22 23:22:08.458 DEBG Write     :1045 deps:[JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:08.458 INFO [lossy] skipping 1048
Sep 22 23:22:08.488 DEBG Write     :1045 deps:[JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:08.489 INFO [lossy] sleeping 1 second
Sep 22 23:22:08.726 DEBG Flush     :1043 extent_limit None deps:[JobId(1042), JobId(1041), JobId(1040)] res:true f:15 g:1
Sep 22 23:22:08.764 DEBG Write     :1044 deps:[JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:08.796 DEBG Write     :1045 deps:[JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:08.797 INFO [lossy] skipping 1046
Sep 22 23:22:08.797 INFO [lossy] skipping 1047
Sep 22 23:22:08.797 INFO [lossy] skipping 1049
Sep 22 23:22:08.797 INFO [lossy] skipping 1052
Sep 22 23:22:08.797 INFO [lossy] skipping 1046
Sep 22 23:22:08.797 INFO [lossy] skipping 1049
Sep 22 23:22:08.797 INFO [lossy] skipping 1052
Sep 22 23:22:08.797 WARN returning error on flush!
Sep 22 23:22:08.797 DEBG Flush     :1046 extent_limit None deps:[JobId(1045), JobId(1044), JobId(1043)] res:false f:16 g:1
Sep 22 23:22:08.797 INFO [lossy] skipping 1049
Sep 22 23:22:08.797 INFO [lossy] skipping 1052
Sep 22 23:22:08.803 DEBG Flush     :1046 extent_limit None deps:[JobId(1045), JobId(1044), JobId(1043)] res:true f:16 g:1
Sep 22 23:22:08.803 INFO [lossy] skipping 1049
Sep 22 23:22:08.803 INFO [lossy] skipping 1049
Sep 22 23:22:08.803 INFO [lossy] skipping 1049
Sep 22 23:22:08.804 WARN returning error on write!
Sep 22 23:22:08.804 DEBG Write     :1047 deps:[JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:08.804 INFO [lossy] skipping 1048
Sep 22 23:22:08.804 WARN 1050 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:08.804 INFO [lossy] skipping 1051
Sep 22 23:22:08.805 WARN returning error on write!
Sep 22 23:22:08.805 DEBG Write     :1047 deps:[JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:08.836 DEBG Write     :1048 deps:[JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:08.837 WARN 1051 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:08.837 INFO [lossy] skipping 1047
Sep 22 23:22:08.867 DEBG Write     :1047 deps:[JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:08.869 INFO [lossy] sleeping 1 second
Sep 22 23:22:09.530 WARN returning error on write!
Sep 22 23:22:09.530 DEBG Write     :1050 deps:[JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:09.562 DEBG Write     :1051 deps:[JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.563 INFO [lossy] skipping 1052
Sep 22 23:22:09.563 INFO [lossy] skipping 1055
Sep 22 23:22:09.563 INFO [lossy] skipping 1056
Sep 22 23:22:09.563 INFO [lossy] skipping 1050
Sep 22 23:22:09.563 INFO [lossy] skipping 1056
Sep 22 23:22:09.594 DEBG Write     :1050 deps:[JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.595 INFO [lossy] skipping 1056
Sep 22 23:22:09.595 WARN returning error on flush!
Sep 22 23:22:09.595 DEBG Flush     :1052 extent_limit None deps:[JobId(1051), JobId(1050), JobId(1049)] res:false f:18 g:1
Sep 22 23:22:09.595 INFO [lossy] skipping 1054
Sep 22 23:22:09.595 INFO [lossy] skipping 1046
Sep 22 23:22:09.595 INFO [lossy] skipping 1049
Sep 22 23:22:09.595 INFO [lossy] skipping 1054
Sep 22 23:22:09.595 INFO [lossy] skipping 1056
Sep 22 23:22:09.608 DEBG Flush     :1052 extent_limit None deps:[JobId(1051), JobId(1050), JobId(1049)] res:true f:18 g:1
Sep 22 23:22:09.608 INFO [lossy] skipping 1054
Sep 22 23:22:09.639 DEBG Write     :1054 deps:[JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.670 DEBG Write     :1053 deps:[JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.672 DEBG Flush     :1046 extent_limit None deps:[JobId(1045), JobId(1044), JobId(1043)] res:true f:16 g:1
Sep 22 23:22:09.672 INFO [lossy] skipping 1049
Sep 22 23:22:09.672 WARN 1049 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:09.703 DEBG Write     :1047 deps:[JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.735 DEBG Write     :1048 deps:[JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.749 DEBG Flush     :1055 extent_limit None deps:[JobId(1054), JobId(1053), JobId(1052)] res:true f:19 g:1
Sep 22 23:22:09.780 DEBG Write     :1056 deps:[JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.781 INFO [lossy] sleeping 1 second
Sep 22 23:22:09.781 DEBG Flush     :1049 extent_limit None deps:[JobId(1048), JobId(1047), JobId(1046)] res:true f:17 g:1
Sep 22 23:22:09.813 DEBG Write     :1050 deps:[JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.814 INFO [lossy] skipping 1051
Sep 22 23:22:09.814 WARN 1052 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:09.814 WARN 1053 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:09.814 WARN 1054 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:09.814 WARN 1056 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:09.846 DEBG Write     :1051 deps:[JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.847 INFO [lossy] skipping 1052
Sep 22 23:22:09.847 WARN returning error on flush!
Sep 22 23:22:09.847 DEBG Flush     :1052 extent_limit None deps:[JobId(1051), JobId(1050), JobId(1049)] res:false f:18 g:1
Sep 22 23:22:09.847 INFO [lossy] skipping 1052
Sep 22 23:22:09.854 DEBG Flush     :1052 extent_limit None deps:[JobId(1051), JobId(1050), JobId(1049)] res:true f:18 g:1
Sep 22 23:22:09.854 INFO [lossy] skipping 1053
Sep 22 23:22:09.854 INFO [lossy] skipping 1054
Sep 22 23:22:09.854 WARN 1055 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:09.854 WARN 1056 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:09.855 WARN returning error on write!
Sep 22 23:22:09.855 DEBG Write     :1053 deps:[JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:09.887 DEBG Write     :1054 deps:[JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.888 INFO [lossy] skipping 1053
Sep 22 23:22:09.889 WARN returning error on write!
Sep 22 23:22:09.889 DEBG Write     :1053 deps:[JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:09.890 WARN returning error on write!
Sep 22 23:22:09.890 DEBG Write     :1053 deps:[JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:09.890 INFO [lossy] skipping 1053
Sep 22 23:22:09.921 DEBG Write     :1053 deps:[JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:09.922 INFO [lossy] skipping 1055
Sep 22 23:22:09.928 DEBG Flush     :1055 extent_limit None deps:[JobId(1054), JobId(1053), JobId(1052)] res:true f:19 g:1
Sep 22 23:22:09.929 INFO [lossy] sleeping 1 second
Sep 22 23:22:09.935 DEBG Flush     :1049 extent_limit None deps:[JobId(1048), JobId(1047), JobId(1046)] res:true f:17 g:1
Sep 22 23:22:09.937 WARN returning error on write!
Sep 22 23:22:09.937 DEBG Write     :1050 deps:[JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:09.938 WARN returning error on write!
Sep 22 23:22:09.938 DEBG Write     :1051 deps:[JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:09.938 WARN 1052 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:09.938 INFO [lossy] skipping 1054
Sep 22 23:22:09.969 DEBG Write     :1050 deps:[JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:10.001 DEBG Write     :1051 deps:[JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:10.002 INFO [lossy] skipping 1054
Sep 22 23:22:10.009 DEBG Flush     :1052 extent_limit None deps:[JobId(1051), JobId(1050), JobId(1049)] res:true f:18 g:1
Sep 22 23:22:10.040 DEBG Write     :1053 deps:[JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:10.072 DEBG Write     :1054 deps:[JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:10.073 INFO [lossy] skipping 1055
Sep 22 23:22:10.073 INFO [lossy] skipping 1056
Sep 22 23:22:10.080 DEBG Flush     :1055 extent_limit None deps:[JobId(1054), JobId(1053), JobId(1052)] res:true f:19 g:1
Sep 22 23:22:10.081 WARN returning error on write!
Sep 22 23:22:10.081 DEBG Write     :1056 deps:[JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:10.081 INFO [lossy] skipping 1056
Sep 22 23:22:10.111 DEBG Write     :1056 deps:[JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:10.113 INFO [lossy] sleeping 1 second
Sep 22 23:22:10.949 INFO [lossy] skipping 1057
Sep 22 23:22:10.981 DEBG Write     :1057 deps:[JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:10.982 INFO [lossy] sleeping 1 second
Sep 22 23:22:11.013 DEBG Write     :1056 deps:[JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:11.045 DEBG Write     :1057 deps:[JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:11.046 WARN returning error on flush!
Sep 22 23:22:11.046 DEBG Flush     :1058 extent_limit None deps:[JobId(1057), JobId(1056), JobId(1055)] res:false f:20 g:1
Sep 22 23:22:11.046 INFO [lossy] skipping 1059
Sep 22 23:22:11.046 INFO [lossy] skipping 1060
Sep 22 23:22:11.046 WARN returning error on flush!
Sep 22 23:22:11.046 DEBG Flush     :1058 extent_limit None deps:[JobId(1057), JobId(1056), JobId(1055)] res:false f:20 g:1
Sep 22 23:22:11.052 DEBG Flush     :1058 extent_limit None deps:[JobId(1057), JobId(1056), JobId(1055)] res:true f:20 g:1
Sep 22 23:22:11.053 WARN returning error on write!
Sep 22 23:22:11.053 DEBG Write     :1059 deps:[JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:11.084 DEBG Write     :1060 deps:[JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:11.085 INFO [lossy] skipping 1061
Sep 22 23:22:11.085 WARN 1062 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:11.115 DEBG Write     :1059 deps:[JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:11.123 DEBG Flush     :1061 extent_limit None deps:[JobId(1060), JobId(1059), JobId(1058)] res:true f:21 g:1
Sep 22 23:22:11.123 INFO [lossy] skipping 1062
Sep 22 23:22:11.123 INFO [lossy] skipping 1062
Sep 22 23:22:11.153 DEBG Write     :1062 deps:[JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:11.154 INFO [lossy] sleeping 1 second
Sep 22 23:22:11.155 WARN returning error on write!
Sep 22 23:22:11.155 DEBG Write     :1057 deps:[JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:11.156 INFO [lossy] skipping 1060
Sep 22 23:22:11.186 DEBG Write     :1057 deps:[JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:11.187 INFO [lossy] skipping 1060
Sep 22 23:22:11.187 INFO [lossy] sleeping 1 second
Sep 22 23:22:12.032 DEBG Flush     :1058 extent_limit None deps:[JobId(1057), JobId(1056), JobId(1055)] res:true f:20 g:1
Sep 22 23:22:12.063 DEBG Write     :1059 deps:[JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.094 DEBG Write     :1060 deps:[JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.101 DEBG Flush     :1061 extent_limit None deps:[JobId(1060), JobId(1059), JobId(1058)] res:true f:21 g:1
Sep 22 23:22:12.102 WARN returning error on write!
Sep 22 23:22:12.102 DEBG Write     :1062 deps:[JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:12.103 WARN returning error on write!
Sep 22 23:22:12.103 DEBG Write     :1063 deps:[JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:12.133 DEBG Write     :1062 deps:[JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.164 DEBG Write     :1063 deps:[JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.172 DEBG Flush     :1064 extent_limit None deps:[JobId(1063), JobId(1062), JobId(1061)] res:true f:22 g:1
Sep 22 23:22:12.172 INFO [lossy] skipping 1065
Sep 22 23:22:12.172 INFO [lossy] skipping 1066
Sep 22 23:22:12.172 WARN 1067 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:12.172 WARN 1068 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:12.202 DEBG Write     :1065 deps:[JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.234 DEBG Write     :1066 deps:[JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.242 DEBG Flush     :1067 extent_limit None deps:[JobId(1066), JobId(1065), JobId(1064)] res:true f:23 g:1
Sep 22 23:22:12.273 DEBG Write     :1068 deps:[JobId(1067), JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.274 INFO [lossy] sleeping 1 second
Sep 22 23:22:12.305 DEBG Write     :1063 deps:[JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.307 INFO [lossy] skipping 1064
Sep 22 23:22:12.307 INFO [lossy] skipping 1066
Sep 22 23:22:12.307 INFO [lossy] skipping 1064
Sep 22 23:22:12.319 DEBG Flush     :1064 extent_limit None deps:[JobId(1063), JobId(1062), JobId(1061)] res:true f:22 g:1
Sep 22 23:22:12.320 WARN returning error on write!
Sep 22 23:22:12.320 DEBG Write     :1065 deps:[JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:12.351 DEBG Write     :1066 deps:[JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.352 INFO [lossy] skipping 1067
Sep 22 23:22:12.352 WARN 1068 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:12.353 WARN returning error on write!
Sep 22 23:22:12.353 DEBG Write     :1065 deps:[JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:12.353 WARN 1067 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:12.353 INFO [lossy] skipping 1065
Sep 22 23:22:12.383 DEBG Write     :1065 deps:[JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.385 DEBG Flush     :1058 extent_limit None deps:[JobId(1057), JobId(1056), JobId(1055)] res:true f:20 g:1
Sep 22 23:22:12.416 DEBG Write     :1059 deps:[JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.448 DEBG Write     :1060 deps:[JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.449 INFO [lossy] skipping 1061
Sep 22 23:22:12.449 INFO [lossy] skipping 1062
Sep 22 23:22:12.449 INFO [lossy] skipping 1067
Sep 22 23:22:12.449 WARN returning error on flush!
Sep 22 23:22:12.449 DEBG Flush     :1061 extent_limit None deps:[JobId(1060), JobId(1059), JobId(1058)] res:false f:21 g:1
Sep 22 23:22:12.449 WARN 1062 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:12.462 DEBG Flush     :1067 extent_limit None deps:[JobId(1066), JobId(1065), JobId(1064)] res:true f:23 g:1
Sep 22 23:22:12.493 DEBG Write     :1068 deps:[JobId(1067), JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.494 INFO [lossy] sleeping 1 second
Sep 22 23:22:12.494 DEBG Flush     :1061 extent_limit None deps:[JobId(1060), JobId(1059), JobId(1058)] res:true f:21 g:1
Sep 22 23:22:12.525 DEBG Write     :1062 deps:[JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.527 WARN returning error on write!
Sep 22 23:22:12.527 DEBG Write     :1063 deps:[JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:false
Sep 22 23:22:12.527 INFO [lossy] skipping 1064
Sep 22 23:22:12.527 INFO [lossy] skipping 1065
Sep 22 23:22:12.527 WARN 1066 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:12.527 INFO [lossy] skipping 1068
Sep 22 23:22:12.527 INFO [lossy] skipping 1063
Sep 22 23:22:12.528 WARN 1064 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:12.528 INFO [lossy] skipping 1065
Sep 22 23:22:12.528 WARN 1068 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:22:12.558 DEBG Write     :1063 deps:[JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.559 WARN 1065 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:12.566 DEBG Flush     :1064 extent_limit None deps:[JobId(1063), JobId(1062), JobId(1061)] res:true f:22 g:1
Sep 22 23:22:12.596 DEBG Write     :1065 deps:[JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.629 DEBG Write     :1066 deps:[JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.636 DEBG Flush     :1067 extent_limit None deps:[JobId(1066), JobId(1065), JobId(1064)] res:true f:23 g:1
Sep 22 23:22:12.636 INFO [lossy] skipping 1068
Sep 22 23:22:12.636 INFO [lossy] skipping 1068
Sep 22 23:22:12.636 INFO [lossy] skipping 1068
Sep 22 23:22:12.636 INFO [lossy] skipping 1068
Sep 22 23:22:12.667 DEBG Write     :1068 deps:[JobId(1067), JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:12.669 INFO [lossy] sleeping 1 second
Sep 22 23:22:12.754 DEBG Write     :1000 deps:[] res:true
Sep 22 23:22:12.822 ERRO [0] job id 1011 saw error GenericError("test error")
Sep 22 23:22:12.822 ERRO [0] job id 1017 saw error GenericError("test error")
Sep 22 23:22:12.822 ERRO [0] job id 1025 saw error GenericError("test error")
Sep 22 23:22:12.822 ERRO [0] job id 1029 saw error GenericError("test error")
Sep 22 23:22:12.823 ERRO [1] job id 1011 saw error GenericError("test error")
Sep 22 23:22:12.823 ERRO [2] job id 1008 saw error GenericError("test error")
Sep 22 23:22:12.823 DEBG up_ds_listen was notified
Sep 22 23:22:12.823 DEBG up_ds_listen process 1010
Sep 22 23:22:12.823 DEBG [A] ack job 1010:11, : downstairs
Sep 22 23:22:12.824 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.824 ERRO [1] job id 1011 saw error GenericError("test error")
Sep 22 23:22:12.824 ERRO [2] job id 1009 saw error GenericError("test error")
Sep 22 23:22:12.824 ERRO [0] job id 1032 saw error GenericError("test error")
Sep 22 23:22:12.824 ERRO [1] job id 1011 saw error GenericError("test error")
Sep 22 23:22:12.824 ERRO [0] job id 1032 saw error GenericError("test error")
Sep 22 23:22:12.826 DEBG [rc] retire 1010 clears [JobId(1008), JobId(1009), JobId(1010)], : downstairs
Sep 22 23:22:12.826 DEBG up_ds_listen was notified
Sep 22 23:22:12.827 DEBG up_ds_listen process 1013
Sep 22 23:22:12.827 DEBG [A] ack job 1013:14, : downstairs
Sep 22 23:22:12.827 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.827 ERRO [0] job id 1033 saw error GenericError("test error")
Sep 22 23:22:12.830 DEBG [rc] retire 1013 clears [JobId(1011), JobId(1012), JobId(1013)], : downstairs
Sep 22 23:22:12.830 ERRO [2] job id 1014 saw error GenericError("test error")
Sep 22 23:22:12.830 DEBG up_ds_listen was notified
Sep 22 23:22:12.830 DEBG up_ds_listen process 1016
Sep 22 23:22:12.830 DEBG [A] ack job 1016:17, : downstairs
Sep 22 23:22:12.830 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.837 ERRO [1] job id 1017 saw error GenericError("test error")
Sep 22 23:22:12.837 ERRO [0] job id 1037 saw error GenericError("test error")
Sep 22 23:22:12.841 DEBG [rc] retire 1016 clears [JobId(1014), JobId(1015), JobId(1016)], : downstairs
Sep 22 23:22:12.841 DEBG up_ds_listen was notified
Sep 22 23:22:12.841 DEBG up_ds_listen process 1019
Sep 22 23:22:12.841 DEBG [A] ack job 1019:20, : downstairs
Sep 22 23:22:12.841 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.841 ERRO [2] job id 1018 saw error GenericError("test error")
Sep 22 23:22:12.841 ERRO [1] job id 1020 saw error GenericError("test error")
Sep 22 23:22:12.841 ERRO [1] job id 1020 saw error GenericError("test error")
Sep 22 23:22:12.844 DEBG [rc] retire 1019 clears [JobId(1017), JobId(1018), JobId(1019)], : downstairs
Sep 22 23:22:12.844 ERRO [0] job id 1041 saw error GenericError("test error")
Sep 22 23:22:12.844 ERRO [1] job id 1020 saw error GenericError("test error")
Sep 22 23:22:12.844 ERRO [0] job id 1042 saw error GenericError("test error")
Sep 22 23:22:12.844 ERRO [1] job id 1022 saw error GenericError("test error")
Sep 22 23:22:12.844 ERRO [2] job id 1022 saw error GenericError("test error")
Sep 22 23:22:12.844 ERRO [0] job id 1042 saw error GenericError("test error")
Sep 22 23:22:12.844 ERRO [1] job id 1022 saw error GenericError("test error")
Sep 22 23:22:12.845 DEBG up_ds_listen was notified
Sep 22 23:22:12.845 DEBG up_ds_listen process 1022
Sep 22 23:22:12.845 DEBG [A] ack job 1022:23, : downstairs
Sep 22 23:22:12.845 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.848 DEBG [rc] retire 1022 clears [JobId(1020), JobId(1021), JobId(1022)], : downstairs
Sep 22 23:22:12.848 ERRO [0] job id 1043 saw error GenericError("test error")
Sep 22 23:22:12.848 ERRO [1] job id 1024 saw error GenericError("test error")
Sep 22 23:22:12.848 DEBG up_ds_listen was notified
Sep 22 23:22:12.848 DEBG up_ds_listen process 1025
Sep 22 23:22:12.848 DEBG [A] ack job 1025:26, : downstairs
Sep 22 23:22:12.848 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.848 ERRO [0] job id 1045 saw error GenericError("test error")
Sep 22 23:22:12.848 ERRO [2] job id 1026 saw error GenericError("test error")
Sep 22 23:22:12.851 DEBG [rc] retire 1025 clears [JobId(1023), JobId(1024), JobId(1025)], : downstairs
Sep 22 23:22:12.851 ERRO [1] job id 1026 saw error GenericError("test error")
Sep 22 23:22:12.851 ERRO [2] job id 1026 saw error GenericError("test error")
Sep 22 23:22:12.851 ERRO [2] job id 1026 saw error GenericError("test error")
Sep 22 23:22:12.851 ERRO [0] job id 1047 saw error GenericError("test error")
Sep 22 23:22:12.852 DEBG up_ds_listen was notified
Sep 22 23:22:12.852 DEBG up_ds_listen process 1028
Sep 22 23:22:12.852 DEBG [A] ack job 1028:29, : downstairs
Sep 22 23:22:12.855 DEBG [rc] retire 1028 clears [JobId(1026), JobId(1027), JobId(1028)], : downstairs
Sep 22 23:22:12.855 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.855 ERRO [1] job id 1029 saw error GenericError("test error")
Sep 22 23:22:12.855 ERRO [0] job id 1050 saw error GenericError("test error")
Sep 22 23:22:12.855 DEBG up_ds_listen was notified
Sep 22 23:22:12.855 DEBG up_ds_listen process 1031
Sep 22 23:22:12.855 DEBG [A] ack job 1031:32, : downstairs
Sep 22 23:22:12.855 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.855 ERRO [1] job id 1031 saw error GenericError("test error")
Sep 22 23:22:12.858 DEBG [rc] retire 1031 clears [JobId(1029), JobId(1030), JobId(1031)], : downstairs
Sep 22 23:22:12.858 ERRO [0] job id 1052 saw error GenericError("test error")
Sep 22 23:22:12.858 DEBG up_ds_listen was notified
Sep 22 23:22:12.858 DEBG up_ds_listen process 1034
Sep 22 23:22:12.858 DEBG [A] ack job 1034:35, : downstairs
Sep 22 23:22:12.858 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.859 ERRO [2] job id 1036 saw error GenericError("test error")
Sep 22 23:22:12.861 DEBG [rc] retire 1034 clears [JobId(1032), JobId(1033), JobId(1034)], : downstairs
Sep 22 23:22:12.861 ERRO [2] job id 1037 saw error GenericError("test error")
Sep 22 23:22:12.862 DEBG up_ds_listen was notified
Sep 22 23:22:12.862 DEBG up_ds_listen process 1037
Sep 22 23:22:12.862 DEBG [A] ack job 1037:38, : downstairs
Sep 22 23:22:12.864 DEBG [rc] retire 1037 clears [JobId(1035), JobId(1036), JobId(1037)], : downstairs
Sep 22 23:22:12.864 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.865 ERRO [2] job id 1040 saw error GenericError("test error")
Sep 22 23:22:12.865 DEBG up_ds_listen was notified
Sep 22 23:22:12.865 DEBG up_ds_listen process 1040
Sep 22 23:22:12.865 DEBG [A] ack job 1040:41, : downstairs
Sep 22 23:22:12.865 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.865 ERRO [1] job id 1041 saw error GenericError("test error")
Sep 22 23:22:12.867 DEBG [rc] retire 1040 clears [JobId(1038), JobId(1039), JobId(1040)], : downstairs
Sep 22 23:22:12.868 ERRO [1] job id 1041 saw error GenericError("test error")
Sep 22 23:22:12.868 ERRO [0] job id 1062 saw error GenericError("test error")
Sep 22 23:22:12.868 ERRO [2] job id 1042 saw error GenericError("test error")
Sep 22 23:22:12.868 ERRO [0] job id 1063 saw error GenericError("test error")
Sep 22 23:22:12.868 ERRO [1] job id 1042 saw error GenericError("test error")
Sep 22 23:22:12.868 ERRO [2] job id 1042 saw error GenericError("test error")
Sep 22 23:22:12.868 ERRO [2] job id 1042 saw error GenericError("test error")
Sep 22 23:22:12.868 DEBG up_ds_listen was notified
Sep 22 23:22:12.868 DEBG up_ds_listen process 1043
Sep 22 23:22:12.868 DEBG [A] ack job 1043:44, : downstairs
Sep 22 23:22:12.868 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.871 DEBG [rc] retire 1043 clears [JobId(1041), JobId(1042), JobId(1043)], : downstairs
Sep 22 23:22:12.871 ERRO [2] job id 1044 saw error GenericError("test error")
Sep 22 23:22:12.871 ERRO [1] job id 1046 saw error GenericError("test error")
Sep 22 23:22:12.871 ERRO [2] job id 1045 saw error GenericError("test error")
Sep 22 23:22:12.871 DEBG up_ds_listen was notified
Sep 22 23:22:12.871 DEBG up_ds_listen process 1046
Sep 22 23:22:12.871 DEBG [A] ack job 1046:47, : downstairs
Sep 22 23:22:12.871 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.871 ERRO [1] job id 1047 saw error GenericError("test error")
Sep 22 23:22:12.871 ERRO [1] job id 1047 saw error GenericError("test error")
Sep 22 23:22:12.874 DEBG [rc] retire 1046 clears [JobId(1044), JobId(1045), JobId(1046)], : downstairs
Sep 22 23:22:12.874 DEBG up_ds_listen was notified
Sep 22 23:22:12.874 DEBG up_ds_listen process 1049
Sep 22 23:22:12.874 DEBG [A] ack job 1049:50, : downstairs
Sep 22 23:22:12.877 DEBG [rc] retire 1049 clears [JobId(1047), JobId(1048), JobId(1049)], : downstairs
Sep 22 23:22:12.877 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.877 ERRO [1] job id 1050 saw error GenericError("test error")
Sep 22 23:22:12.877 ERRO [1] job id 1051 saw error GenericError("test error")
Sep 22 23:22:12.877 ERRO [2] job id 1052 saw error GenericError("test error")
Sep 22 23:22:12.877 DEBG up_ds_listen was notified
Sep 22 23:22:12.877 DEBG up_ds_listen process 1052
Sep 22 23:22:12.877 DEBG [A] ack job 1052:53, : downstairs
Sep 22 23:22:12.877 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.877 ERRO [2] job id 1053 saw error GenericError("test error")
Sep 22 23:22:12.880 DEBG [rc] retire 1052 clears [JobId(1050), JobId(1051), JobId(1052)], : downstairs
Sep 22 23:22:12.880 ERRO [2] job id 1053 saw error GenericError("test error")
Sep 22 23:22:12.880 ERRO [2] job id 1053 saw error GenericError("test error")
Sep 22 23:22:12.880 DEBG up_ds_listen was notified
Sep 22 23:22:12.880 DEBG up_ds_listen process 1055
Sep 22 23:22:12.880 DEBG [A] ack job 1055:56, : downstairs
Sep 22 23:22:12.880 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.880 ERRO [1] job id 1056 saw error GenericError("test error")
Sep 22 23:22:12.883 DEBG [rc] retire 1055 clears [JobId(1053), JobId(1054), JobId(1055)], : downstairs
Sep 22 23:22:12.884 ERRO [1] job id 1057 saw error GenericError("test error")
Sep 22 23:22:12.884 ERRO [2] job id 1058 saw error GenericError("test error")
Sep 22 23:22:12.884 DEBG up_ds_listen was notified
Sep 22 23:22:12.884 DEBG up_ds_listen process 1058
Sep 22 23:22:12.884 DEBG [A] ack job 1058:59, : downstairs
Sep 22 23:22:12.884 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.884 ERRO [2] job id 1058 saw error GenericError("test error")
Sep 22 23:22:12.887 DEBG [rc] retire 1058 clears [JobId(1056), JobId(1057), JobId(1058)], : downstairs
Sep 22 23:22:12.887 ERRO [1] job id 1061 saw error GenericError("test error")
Sep 22 23:22:12.887 ERRO [2] job id 1059 saw error GenericError("test error")
Sep 22 23:22:12.887 DEBG up_ds_listen was notified
Sep 22 23:22:12.887 DEBG up_ds_listen process 1061
Sep 22 23:22:12.887 DEBG [A] ack job 1061:62, : downstairs
Sep 22 23:22:12.887 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.887 ERRO [1] job id 1063 saw error GenericError("test error")
Sep 22 23:22:12.890 DEBG [rc] retire 1061 clears [JobId(1059), JobId(1060), JobId(1061)], : downstairs
Sep 22 23:22:12.890 DEBG up_ds_listen was notified
Sep 22 23:22:12.890 DEBG up_ds_listen process 1064
Sep 22 23:22:12.890 DEBG [A] ack job 1064:65, : downstairs
Sep 22 23:22:12.890 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.893 DEBG [rc] retire 1064 clears [JobId(1062), JobId(1063), JobId(1064)], : downstairs
Sep 22 23:22:12.893 ERRO [2] job id 1065 saw error GenericError("test error")
Sep 22 23:22:12.893 DEBG up_ds_listen was notified
Sep 22 23:22:12.893 DEBG up_ds_listen process 1067
Sep 22 23:22:12.893 DEBG [A] ack job 1067:68, : downstairs
Sep 22 23:22:12.893 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:12.893 ERRO [2] job id 1065 saw error GenericError("test error")
Sep 22 23:22:12.896 DEBG [rc] retire 1067 clears [JobId(1065), JobId(1066), JobId(1067)], : downstairs
Sep 22 23:22:13.195 DEBG Write     :1000 deps:[] res:true
Sep 22 23:22:13.307 DEBG Write     :1069 deps:[JobId(1067), JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:13.308 WARN returning error on flush!
Sep 22 23:22:13.308 DEBG Flush     :1070 extent_limit None deps:[JobId(1069), JobId(1068), JobId(1067)] res:false f:24 g:1
Sep 22 23:22:13.308 INFO [lossy] skipping 1071
Sep 22 23:22:13.308 INFO [lossy] skipping 1070
Sep 22 23:22:13.315 DEBG Flush     :1070 extent_limit None deps:[JobId(1069), JobId(1068), JobId(1067)] res:true f:24 g:1
Sep 22 23:22:13.321 DEBG Read      :1071 deps:[JobId(1070)] res:true
Sep 22 23:22:13.343 ERRO [0] job id 1070 saw error GenericError("test error")
Sep 22 23:22:13.345 DEBG Flush     :1072 extent_limit None deps:[JobId(1071), JobId(1070)] res:true f:25 g:1
Sep 22 23:22:13.345 INFO [lossy] sleeping 1 second
Sep 22 23:22:13.636 DEBG Write     :1000 deps:[] res:true
Sep 22 23:22:13.727 DEBG [0] Read AckReady 1071, : downstairs
Sep 22 23:22:13.728 DEBG up_ds_listen was notified
Sep 22 23:22:13.728 DEBG up_ds_listen process 1071
Sep 22 23:22:13.729 DEBG [A] ack job 1071:72, : downstairs
Sep 22 23:22:13.783 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:13.815 DEBG Write     :1069 deps:[JobId(1067), JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:13.816 INFO [lossy] skipping 1069
Sep 22 23:22:13.816 INFO [lossy] skipping 1071
Sep 22 23:22:13.847 DEBG Write     :1069 deps:[JobId(1067), JobId(1064), JobId(1061), JobId(1058), JobId(1055), JobId(1052), JobId(1049), JobId(1046), JobId(1043), JobId(1040), JobId(1037), JobId(1034), JobId(1031), JobId(1028), JobId(1025), JobId(1022), JobId(1019), JobId(1016), JobId(1013), JobId(1010), JobId(1007), JobId(1004), JobId(1001)] res:true
Sep 22 23:22:13.848 WARN returning error on flush!
Sep 22 23:22:13.848 DEBG Flush     :1070 extent_limit None deps:[JobId(1069), JobId(1068), JobId(1067)] res:false f:24 g:1
Sep 22 23:22:13.848 INFO [lossy] skipping 1072
Sep 22 23:22:13.848 INFO [lossy] skipping 1070
Sep 22 23:22:13.861 DEBG Flush     :1070 extent_limit None deps:[JobId(1069), JobId(1068), JobId(1067)] res:true f:24 g:1
Sep 22 23:22:13.861 INFO [lossy] skipping 1071
Sep 22 23:22:13.861 INFO [lossy] skipping 1071
Sep 22 23:22:13.867 DEBG Read      :1071 deps:[JobId(1070)] res:true
Sep 22 23:22:13.888 DEBG Flush     :1070 extent_limit None deps:[JobId(1069), JobId(1068), JobId(1067)] res:true f:24 g:1
Sep 22 23:22:13.895 DEBG Read      :1071 deps:[JobId(1070)] res:true
Sep 22 23:22:13.917 DEBG IO Read  1073 has deps [JobId(1072)]
Sep 22 23:22:13.917 ERRO [1] job id 1070 saw error GenericError("test error")
Sep 22 23:22:13.923 DEBG up_ds_listen was notified
Sep 22 23:22:13.923 DEBG up_ds_listen process 1070
Sep 22 23:22:13.923 DEBG [A] ack job 1070:71, : downstairs
Sep 22 23:22:13.924 WARN 9144e02c-c312-47c4-9b1c-f03618834608 request to replace downstairs 127.0.0.1:52165 with 127.0.0.1:42129
Sep 22 23:22:13.924 INFO 9144e02c-c312-47c4-9b1c-f03618834608 found new target: 127.0.0.1:42129 at 0
Sep 22 23:22:13.925 DEBG [rc] retire 1070 clears [JobId(1068), JobId(1069), JobId(1070)], : downstairs
Sep 22 23:22:13.925 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:13.934 DEBG Flush     :1072 extent_limit None deps:[JobId(1071), JobId(1070)] res:true f:25 g:1
Sep 22 23:22:13.934 INFO [lossy] sleeping 1 second
Waited for some repair work, proceeding with test
Sep 22 23:22:13.936 INFO [lossy] sleeping 1 second
Sep 22 23:22:13.966 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:22:14.008 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:1 g:1
Sep 22 23:22:14.008 WARN [0] will exit pm_task, this downstairs Replacing
Sep 22 23:22:14.008 ERRO 127.0.0.1:52165: proc: [0] client work task ended, Ok(Err([0] This downstairs now in Replacing)), so we end too, looper: 0
Sep 22 23:22:14.008 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 Gone missing, transition from Replacing to Replaced
Sep 22 23:22:14.008 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 connection to 127.0.0.1:52165 closed, looper: 0
Sep 22 23:22:14.008 INFO [0] 127.0.0.1:52165 task reports connection:false
Sep 22 23:22:14.008 INFO 9144e02c-c312-47c4-9b1c-f03618834608 Replaced Active Active
Sep 22 23:22:14.008 INFO [0] 127.0.0.1:52165 task reports offline
Sep 22 23:22:14.009 INFO Upstairs starts
Sep 22 23:22:14.009 INFO Crucible Version: BuildInfo {
    version: "0.0.1",
    git_sha: "ed48f294784d46ea7d4bb99336918b74358eca46",
    git_commit_timestamp: "2023-09-22T22:51:18.000000000Z",
    git_branch: "main",
    rustc_semver: "1.70.0",
    rustc_channel: "stable",
    rustc_host_triple: "x86_64-unknown-illumos",
    rustc_commit_sha: "90c541806f23a127002de5b4038be731ba1458ca",
    cargo_triple: "x86_64-unknown-illumos",
    debug: true,
    opt_level: 0,
}
Sep 22 23:22:14.009 INFO Upstairs <-> Downstairs Message Version: 4
Sep 22 23:22:14.009 INFO Crucible stats registered with UUID: 9144e02c-c312-47c4-9b1c-f03618834608
Sep 22 23:22:14.009 INFO Crucible 9144e02c-c312-47c4-9b1c-f03618834608 has session id: 163c9a5f-e237-4960-a267-a823f80aaaa9
Sep 22 23:22:14.009 WARN upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } disconnected, 0 jobs left, task: main
Sep 22 23:22:14.009 WARN upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } was previously active, clearing, task: main
Sep 22 23:22:14.009 INFO connection (127.0.0.1:33221): all done
Sep 22 23:22:14.009 INFO [0] connecting to 127.0.0.1:42129, looper: 0
Sep 22 23:22:14.009 INFO [1] connecting to 127.0.0.1:48339, looper: 1
Sep 22 23:22:14.009 INFO [2] connecting to 127.0.0.1:33021, looper: 2
Sep 22 23:22:14.009 INFO up_listen starts, task: up_listen
Sep 22 23:22:14.009 INFO Wait for all three downstairs to come online
Sep 22 23:22:14.009 INFO Flush timeout: 0.5
Sep 22 23:22:14.009 DEBG up_ds_listen was notified
Sep 22 23:22:14.010 DEBG up_ds_listen process 1001
Sep 22 23:22:14.010 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:22:14.079 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:22:14.079 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:14.079 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 looper connected, looper: 0
Sep 22 23:22:14.079 INFO [0] Proc runs for 127.0.0.1:42129 in state New
Sep 22 23:22:14.079 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 looper connected, looper: 1
Sep 22 23:22:14.079 INFO [1] Proc runs for 127.0.0.1:48339 in state New
Sep 22 23:22:14.079 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 looper connected, looper: 2
Sep 22 23:22:14.079 INFO [2] Proc runs for 127.0.0.1:33021 in state New
Sep 22 23:22:14.079 INFO accepted connection from 127.0.0.1:33177, task: main
Sep 22 23:22:14.079 INFO accepted connection from 127.0.0.1:52771, task: main
Sep 22 23:22:14.079 INFO accepted connection from 127.0.0.1:38863, task: main
Sep 22 23:22:14.080 INFO Connection request from 9144e02c-c312-47c4-9b1c-f03618834608 with version 4, task: proc
Sep 22 23:22:14.080 INFO upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 } connected, version 4, task: proc
Sep 22 23:22:14.080 INFO Connection request from 9144e02c-c312-47c4-9b1c-f03618834608 with version 4, task: proc
Sep 22 23:22:14.080 INFO upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 } connected, version 4, task: proc
Sep 22 23:22:14.080 INFO Connection request from 9144e02c-c312-47c4-9b1c-f03618834608 with version 4, task: proc
Sep 22 23:22:14.080 INFO upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 } connected, version 4, task: proc
The guest has requested activation
Sep 22 23:22:14.080 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 (ec8dc28c-8140-4e2d-b538-f84f8abf147b) New New New ds_transition to WaitActive
Sep 22 23:22:14.080 INFO [0] Transition from New to WaitActive
Sep 22 23:22:14.080 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 (ec8dc28c-8140-4e2d-b538-f84f8abf147b) WaitActive New New ds_transition to WaitActive
Sep 22 23:22:14.080 INFO [1] Transition from New to WaitActive
Sep 22 23:22:14.080 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 (ec8dc28c-8140-4e2d-b538-f84f8abf147b) WaitActive WaitActive New ds_transition to WaitActive
Sep 22 23:22:14.080 INFO [2] Transition from New to WaitActive
Sep 22 23:22:14.080 INFO 9144e02c-c312-47c4-9b1c-f03618834608 active request set
Sep 22 23:22:14.080 INFO [0] received activate with gen 2
Sep 22 23:22:14.080 INFO [0] client got ds_active_rx, promote! session ec8dc28c-8140-4e2d-b538-f84f8abf147b
Sep 22 23:22:14.080 INFO [1] received activate with gen 2
Sep 22 23:22:14.080 INFO [1] client got ds_active_rx, promote! session ec8dc28c-8140-4e2d-b538-f84f8abf147b
Sep 22 23:22:14.081 INFO [2] received activate with gen 2
Sep 22 23:22:14.081 INFO [2] client got ds_active_rx, promote! session ec8dc28c-8140-4e2d-b538-f84f8abf147b
Sep 22 23:22:14.081 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } to UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 }
Sep 22 23:22:14.081 WARN Signaling to UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } thread that UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 } is being promoted (read-write)
Sep 22 23:22:14.081 WARN Attempting RW takeover from UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } to UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 }
Sep 22 23:22:14.081 WARN Signaling to UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } thread that UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 } is being promoted (read-write)
Sep 22 23:22:14.081 INFO UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 } is now active (read-write)
Sep 22 23:22:14.081 WARN Another upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 }, task: main
Sep 22 23:22:14.081 INFO UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 } is now active (read-write)
Sep 22 23:22:14.081 WARN Another upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 } promoted to active, shutting down connection for UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 }, task: main
Sep 22 23:22:14.082 INFO UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 } is now active (read-write)
Sep 22 23:22:14.082 INFO connection (127.0.0.1:35948): all done
Sep 22 23:22:14.082 INFO connection (127.0.0.1:45718): all done
Sep 22 23:22:14.082 ERRO [1] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) cmd_loop saw YouAreNoLongerActive 9144e02c-c312-47c4-9b1c-f03618834608 ec8dc28c-8140-4e2d-b538-f84f8abf147b 2
Sep 22 23:22:14.082 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) Replaced Active Active ds_transition to Disabled
Sep 22 23:22:14.082 INFO [1] Transition from Active to Disabled
Sep 22 23:22:14.082 INFO 9144e02c-c312-47c4-9b1c-f03618834608 set inactive, session cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f
Sep 22 23:22:14.082 ERRO 127.0.0.1:48339: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1, looper: 1
Sep 22 23:22:14.082 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 Gone missing, transition from Disabled to Disconnected
Sep 22 23:22:14.082 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 connection to 127.0.0.1:48339 closed, looper: 1
Sep 22 23:22:14.082 ERRO [2] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) cmd_loop saw YouAreNoLongerActive 9144e02c-c312-47c4-9b1c-f03618834608 ec8dc28c-8140-4e2d-b538-f84f8abf147b 2
Sep 22 23:22:14.082 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) Replaced Disconnected Active ds_transition to Disabled
Sep 22 23:22:14.082 INFO [2] Transition from Active to Disabled
Sep 22 23:22:14.082 INFO 9144e02c-c312-47c4-9b1c-f03618834608 set inactive, session cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f
Sep 22 23:22:14.082 ERRO 127.0.0.1:33021: proc: Generation number is too low: saw YouAreNoLongerActive with larger gen 2 than ours 1, looper: 2
Sep 22 23:22:14.082 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 Gone missing, transition from Disabled to Disconnected
Sep 22 23:22:14.082 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 connection to 127.0.0.1:33021 closed, looper: 2
Sep 22 23:22:14.082 WARN [1] pm_task rx.recv() is None
Sep 22 23:22:14.083 INFO [1] 127.0.0.1:48339 task reports connection:false
Sep 22 23:22:14.083 INFO 9144e02c-c312-47c4-9b1c-f03618834608 Replaced Disconnected Disconnected
Sep 22 23:22:14.083 INFO [1] 127.0.0.1:48339 task reports offline
Sep 22 23:22:14.083 INFO [2] 127.0.0.1:33021 task reports connection:false
Sep 22 23:22:14.083 INFO 9144e02c-c312-47c4-9b1c-f03618834608 Replaced Disconnected Disconnected
Sep 22 23:22:14.083 INFO [2] 127.0.0.1:33021 task reports offline
Sep 22 23:22:14.083 WARN [2] pm_task rx.recv() is None
Sep 22 23:22:14.083 INFO [0] downstairs client at 127.0.0.1:42129 has UUID c3c61ac0-c12a-42ef-b3a2-a316e079f741
Sep 22 23:22:14.083 INFO [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: c3c61ac0-c12a-42ef-b3a2-a316e079f741, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:22:14.083 INFO 9144e02c-c312-47c4-9b1c-f03618834608 WaitActive WaitActive WaitActive
Sep 22 23:22:14.083 INFO [1] downstairs client at 127.0.0.1:48339 has UUID 0b1643bf-b244-4616-9246-4922ec6009af
Sep 22 23:22:14.083 INFO [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 0b1643bf-b244-4616-9246-4922ec6009af, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:22:14.083 INFO 9144e02c-c312-47c4-9b1c-f03618834608 WaitActive WaitActive WaitActive
Sep 22 23:22:14.083 INFO [2] downstairs client at 127.0.0.1:33021 has UUID 99fe08ae-482d-4baf-90c5-1e5bdfb86144
Sep 22 23:22:14.083 INFO [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 512, shift: 9 }, extent_count: 188, uuid: 99fe08ae-482d-4baf-90c5-1e5bdfb86144, encrypted: true, database_read_version: 1, database_write_version: 1 }
Sep 22 23:22:14.083 INFO 9144e02c-c312-47c4-9b1c-f03618834608 WaitActive WaitActive WaitActive
Sep 22 23:22:14.093 INFO Current flush_numbers [0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:22:14.094 INFO Current flush_numbers [0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:22:14.095 INFO Current flush_numbers [0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:22:14.108 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:22:14.109 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:22:14.110 INFO Downstairs has completed Negotiation, task: proc
Sep 22 23:22:14.110 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 (ec8dc28c-8140-4e2d-b538-f84f8abf147b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:22:14.110 INFO [0] Transition from WaitActive to WaitQuorum
Sep 22 23:22:14.110 WARN [0] new RM replaced this: None
Sep 22 23:22:14.110 INFO [0] Starts reconcile loop
Sep 22 23:22:14.110 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 (ec8dc28c-8140-4e2d-b538-f84f8abf147b) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum
Sep 22 23:22:14.110 INFO [1] Transition from WaitActive to WaitQuorum
Sep 22 23:22:14.110 WARN [1] new RM replaced this: None
Sep 22 23:22:14.110 INFO [1] Starts reconcile loop
Sep 22 23:22:14.111 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 (ec8dc28c-8140-4e2d-b538-f84f8abf147b) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum
Sep 22 23:22:14.111 INFO [2] Transition from WaitActive to WaitQuorum
Sep 22 23:22:14.111 WARN [2] new RM replaced this: None
Sep 22 23:22:14.111 INFO [2] Starts reconcile loop
Sep 22 23:22:14.111 INFO [0] 127.0.0.1:42129 task reports connection:true
Sep 22 23:22:14.111 INFO 9144e02c-c312-47c4-9b1c-f03618834608 WaitQuorum WaitQuorum WaitQuorum
Sep 22 23:22:14.111 INFO [0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:22:14.111 INFO [0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
Sep 22 23:22:14.111 INFO [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:22:14.111 INFO [1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:22:14.111 INFO [1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:22:14.111 INFO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:22:14.111 INFO [2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:22:14.111 INFO [2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
Sep 22 23:22:14.111 INFO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
Sep 22 23:22:14.111 INFO Max found gen is 2
Sep 22 23:22:14.111 INFO Generation requested: 2 >= found:2
Sep 22 23:22:14.111 INFO Next flush: 2
Sep 22 23:22:14.111 INFO Extent 0 has flush number mismatch, : mend
Sep 22 23:22:14.111 INFO First source client ID for extent 0, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:0  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:0  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:0  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO find dest for source 1 for extent at index 0, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO Extent 1 has flush number mismatch, : mend
Sep 22 23:22:14.111 INFO First source client ID for extent 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:1  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:1  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:1  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO find dest for source 1 for extent at index 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO Extent 2 has flush number mismatch, : mend
Sep 22 23:22:14.111 INFO First source client ID for extent 2, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:2  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:2  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:2  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO find dest for source 1 for extent at index 2, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO Extent 3 has flush number mismatch, : mend
Sep 22 23:22:14.111 INFO First source client ID for extent 3, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:3  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:3  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:3  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO find dest for source 1 for extent at index 3, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO Extent 4 has flush number mismatch, : mend
Sep 22 23:22:14.111 INFO First source client ID for extent 4, mrl: flush_mismatch, : mend
Sep 22 23:22:14.111 INFO extent:4  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:4  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:4  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO find dest for source 1 for extent at index 4, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO Extent 5 has flush number mismatch, : mend
Sep 22 23:22:14.112 INFO First source client ID for extent 5, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:5  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:5  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:5  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO find dest for source 1 for extent at index 5, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO Extent 6 has flush number mismatch, : mend
Sep 22 23:22:14.112 INFO First source client ID for extent 6, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:6  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:6  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:6  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO find dest for source 1 for extent at index 6, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO Extent 7 has flush number mismatch, : mend
Sep 22 23:22:14.112 INFO First source client ID for extent 7, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:7  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:7  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:7  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO find dest for source 1 for extent at index 7, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO Extent 8 has flush number mismatch, : mend
Sep 22 23:22:14.112 INFO First source client ID for extent 8, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:8  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:8  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:8  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO find dest for source 1 for extent at index 8, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO Extent 9 has flush number mismatch, : mend
Sep 22 23:22:14.112 INFO First source client ID for extent 9, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:9  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:9  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:9  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO find dest for source 1 for extent at index 9, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO Extent 10 has flush number mismatch, : mend
Sep 22 23:22:14.112 INFO First source client ID for extent 10, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:10  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:10  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:10  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO find dest for source 1 for extent at index 10, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO Extent 11 has flush number mismatch, : mend
Sep 22 23:22:14.112 INFO First source client ID for extent 11, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:11  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:11  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:11  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO find dest for source 1 for extent at index 11, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO Extent 12 has flush number mismatch, : mend
Sep 22 23:22:14.112 INFO First source client ID for extent 12, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:12  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:12  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:12  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO find dest for source 1 for extent at index 12, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO Extent 13 has flush number mismatch, : mend
Sep 22 23:22:14.112 INFO First source client ID for extent 13, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:13  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:13  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:13  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO find dest for source 1 for extent at index 13, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO Extent 14 has flush number mismatch, : mend
Sep 22 23:22:14.112 INFO First source client ID for extent 14, mrl: flush_mismatch, : mend
Sep 22 23:22:14.112 INFO extent:14  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:14  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:14  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO find dest for source 1 for extent at index 14, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO Extent 15 has flush number mismatch, : mend
Sep 22 23:22:14.113 INFO First source client ID for extent 15, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:15  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:15  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:15  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO find dest for source 1 for extent at index 15, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO Extent 16 has flush number mismatch, : mend
Sep 22 23:22:14.113 INFO First source client ID for extent 16, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:16  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:16  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:16  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO find dest for source 1 for extent at index 16, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO Extent 17 has flush number mismatch, : mend
Sep 22 23:22:14.113 INFO First source client ID for extent 17, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:17  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:17  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:17  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO find dest for source 1 for extent at index 17, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO Extent 18 has flush number mismatch, : mend
Sep 22 23:22:14.113 INFO First source client ID for extent 18, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:18  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:18  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:18  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO find dest for source 1 for extent at index 18, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO Extent 19 has flush number mismatch, : mend
Sep 22 23:22:14.113 INFO First source client ID for extent 19, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:19  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:19  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:19  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO find dest for source 1 for extent at index 19, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO Extent 20 has flush number mismatch, : mend
Sep 22 23:22:14.113 INFO First source client ID for extent 20, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:20  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:20  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:20  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO find dest for source 1 for extent at index 20, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO Extent 21 has flush number mismatch, : mend
Sep 22 23:22:14.113 INFO First source client ID for extent 21, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:21  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:21  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:21  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO find dest for source 1 for extent at index 21, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO Extent 22 has flush number mismatch, : mend
Sep 22 23:22:14.113 INFO First source client ID for extent 22, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:22  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:22  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:22  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO find dest for source 1 for extent at index 22, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO Extent 23 has flush number mismatch, : mend
Sep 22 23:22:14.113 INFO First source client ID for extent 23, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:23  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:23  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:23  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO find dest for source 1 for extent at index 23, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO Extent 24 has flush number mismatch, : mend
Sep 22 23:22:14.113 INFO First source client ID for extent 24, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:24  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.113 INFO extent:24  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:24  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO find dest for source 1 for extent at index 24, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO Extent 25 has flush number mismatch, : mend
Sep 22 23:22:14.114 INFO First source client ID for extent 25, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:25  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:25  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:25  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO find dest for source 1 for extent at index 25, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO Extent 26 has flush number mismatch, : mend
Sep 22 23:22:14.114 INFO First source client ID for extent 26, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:26  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:26  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:26  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO find dest for source 1 for extent at index 26, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO Extent 27 has flush number mismatch, : mend
Sep 22 23:22:14.114 INFO First source client ID for extent 27, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:27  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:27  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:27  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO find dest for source 1 for extent at index 27, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO Extent 28 has flush number mismatch, : mend
Sep 22 23:22:14.114 INFO First source client ID for extent 28, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:28  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:28  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:28  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO find dest for source 1 for extent at index 28, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO Extent 29 has flush number mismatch, : mend
Sep 22 23:22:14.114 INFO First source client ID for extent 29, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:29  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:29  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:29  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO find dest for source 1 for extent at index 29, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO Extent 30 has flush number mismatch, : mend
Sep 22 23:22:14.114 INFO First source client ID for extent 30, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:30  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:30  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:30  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO find dest for source 1 for extent at index 30, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO Extent 31 has flush number mismatch, : mend
Sep 22 23:22:14.114 INFO First source client ID for extent 31, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:31  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:31  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:31  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO find dest for source 1 for extent at index 31, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO Extent 32 has flush number mismatch, : mend
Sep 22 23:22:14.114 INFO First source client ID for extent 32, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:32  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:32  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:32  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO find dest for source 1 for extent at index 32, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO Extent 33 has flush number mismatch, : mend
Sep 22 23:22:14.114 INFO First source client ID for extent 33, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:33  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:33  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:33  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO find dest for source 1 for extent at index 33, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO Extent 34 has flush number mismatch, : mend
Sep 22 23:22:14.114 INFO First source client ID for extent 34, mrl: flush_mismatch, : mend
Sep 22 23:22:14.114 INFO extent:34  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:34  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:34  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO find dest for source 1 for extent at index 34, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO Extent 35 has flush number mismatch, : mend
Sep 22 23:22:14.115 INFO First source client ID for extent 35, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:35  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:35  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:35  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO find dest for source 1 for extent at index 35, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO Extent 36 has flush number mismatch, : mend
Sep 22 23:22:14.115 INFO First source client ID for extent 36, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:36  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:36  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:36  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO find dest for source 1 for extent at index 36, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO Extent 37 has flush number mismatch, : mend
Sep 22 23:22:14.115 INFO First source client ID for extent 37, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:37  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:37  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:37  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO find dest for source 1 for extent at index 37, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO Extent 38 has flush number mismatch, : mend
Sep 22 23:22:14.115 INFO First source client ID for extent 38, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:38  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:38  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:38  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO find dest for source 1 for extent at index 38, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO Extent 39 has flush number mismatch, : mend
Sep 22 23:22:14.115 INFO First source client ID for extent 39, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:39  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:39  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:39  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO find dest for source 1 for extent at index 39, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO Extent 40 has flush number mismatch, : mend
Sep 22 23:22:14.115 INFO First source client ID for extent 40, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:40  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:40  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:40  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO find dest for source 1 for extent at index 40, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO Extent 41 has flush number mismatch, : mend
Sep 22 23:22:14.115 INFO First source client ID for extent 41, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:41  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:41  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:41  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO find dest for source 1 for extent at index 41, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO Extent 42 has flush number mismatch, : mend
Sep 22 23:22:14.115 INFO First source client ID for extent 42, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:42  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:42  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:42  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO find dest for source 1 for extent at index 42, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO Extent 43 has flush number mismatch, : mend
Sep 22 23:22:14.115 INFO First source client ID for extent 43, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:43  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:43  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:43  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO find dest for source 1 for extent at index 43, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO Extent 44 has flush number mismatch, : mend
Sep 22 23:22:14.115 INFO First source client ID for extent 44, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:44  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.115 INFO extent:44  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:44  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO find dest for source 1 for extent at index 44, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO Extent 45 has flush number mismatch, : mend
Sep 22 23:22:14.116 INFO First source client ID for extent 45, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:45  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:45  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:45  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO find dest for source 1 for extent at index 45, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO Extent 46 has flush number mismatch, : mend
Sep 22 23:22:14.116 INFO First source client ID for extent 46, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:46  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:46  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:46  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO find dest for source 1 for extent at index 46, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO Extent 47 has flush number mismatch, : mend
Sep 22 23:22:14.116 INFO First source client ID for extent 47, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:47  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:47  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:47  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO find dest for source 1 for extent at index 47, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO Extent 48 has flush number mismatch, : mend
Sep 22 23:22:14.116 INFO First source client ID for extent 48, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:48  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:48  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:48  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO find dest for source 1 for extent at index 48, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO Extent 49 has flush number mismatch, : mend
Sep 22 23:22:14.116 INFO First source client ID for extent 49, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:49  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:49  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:49  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO find dest for source 1 for extent at index 49, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO Extent 50 has flush number mismatch, : mend
Sep 22 23:22:14.116 INFO First source client ID for extent 50, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:50  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:50  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:50  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO find dest for source 1 for extent at index 50, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO Extent 51 has flush number mismatch, : mend
Sep 22 23:22:14.116 INFO First source client ID for extent 51, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:51  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:51  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:51  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO find dest for source 1 for extent at index 51, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO Extent 52 has flush number mismatch, : mend
Sep 22 23:22:14.116 INFO First source client ID for extent 52, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:52  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:52  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:52  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO find dest for source 1 for extent at index 52, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO Extent 53 has flush number mismatch, : mend
Sep 22 23:22:14.116 INFO First source client ID for extent 53, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:53  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:53  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:53  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO find dest for source 1 for extent at index 53, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO Extent 54 has flush number mismatch, : mend
Sep 22 23:22:14.116 INFO First source client ID for extent 54, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:54  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO extent:54  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.116 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:54  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO find dest for source 1 for extent at index 54, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO Extent 55 has flush number mismatch, : mend
Sep 22 23:22:14.117 INFO First source client ID for extent 55, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:55  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:55  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:55  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO find dest for source 1 for extent at index 55, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO Extent 56 has flush number mismatch, : mend
Sep 22 23:22:14.117 INFO First source client ID for extent 56, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:56  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:56  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:56  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO find dest for source 1 for extent at index 56, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO Extent 57 has flush number mismatch, : mend
Sep 22 23:22:14.117 INFO First source client ID for extent 57, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:57  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:57  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:57  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO find dest for source 1 for extent at index 57, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO Extent 58 has flush number mismatch, : mend
Sep 22 23:22:14.117 INFO First source client ID for extent 58, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:58  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:58  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:58  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO find dest for source 1 for extent at index 58, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO Extent 59 has flush number mismatch, : mend
Sep 22 23:22:14.117 INFO First source client ID for extent 59, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:59  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:59  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:59  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO find dest for source 1 for extent at index 59, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO Extent 60 has flush number mismatch, : mend
Sep 22 23:22:14.117 INFO First source client ID for extent 60, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:60  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:60  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:60  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO find dest for source 1 for extent at index 60, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO Extent 61 has flush number mismatch, : mend
Sep 22 23:22:14.117 INFO First source client ID for extent 61, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:61  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:61  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:61  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO find dest for source 1 for extent at index 61, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO Extent 62 has flush number mismatch, : mend
Sep 22 23:22:14.117 INFO First source client ID for extent 62, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:62  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:62  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:62  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO find dest for source 1 for extent at index 62, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO Extent 63 has flush number mismatch, : mend
Sep 22 23:22:14.117 INFO First source client ID for extent 63, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:63  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:63  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO extent:63  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO find dest for source 1 for extent at index 63, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.117 INFO Extent 64 has flush number mismatch, : mend
Sep 22 23:22:14.117 INFO First source client ID for extent 64, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:64  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:64  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:64  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO find dest for source 1 for extent at index 64, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO Extent 65 has flush number mismatch, : mend
Sep 22 23:22:14.118 INFO First source client ID for extent 65, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:65  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:65  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:65  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO find dest for source 1 for extent at index 65, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO Extent 66 has flush number mismatch, : mend
Sep 22 23:22:14.118 INFO First source client ID for extent 66, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:66  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:66  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:66  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO find dest for source 1 for extent at index 66, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO Extent 67 has flush number mismatch, : mend
Sep 22 23:22:14.118 INFO First source client ID for extent 67, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:67  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:67  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:67  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO find dest for source 1 for extent at index 67, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO Extent 68 has flush number mismatch, : mend
Sep 22 23:22:14.118 INFO First source client ID for extent 68, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:68  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:68  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:68  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO find dest for source 1 for extent at index 68, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO Extent 69 has flush number mismatch, : mend
Sep 22 23:22:14.118 INFO First source client ID for extent 69, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:69  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:69  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:69  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO find dest for source 1 for extent at index 69, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO Extent 70 has flush number mismatch, : mend
Sep 22 23:22:14.118 INFO First source client ID for extent 70, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:70  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:70  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:70  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO find dest for source 1 for extent at index 70, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO Extent 71 has flush number mismatch, : mend
Sep 22 23:22:14.118 INFO First source client ID for extent 71, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:71  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:71  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:71  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO find dest for source 1 for extent at index 71, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO Extent 72 has flush number mismatch, : mend
Sep 22 23:22:14.118 INFO First source client ID for extent 72, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:72  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:72  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:72  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO find dest for source 1 for extent at index 72, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO Extent 73 has flush number mismatch, : mend
Sep 22 23:22:14.118 INFO First source client ID for extent 73, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:73  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:73  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO extent:73  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO find dest for source 1 for extent at index 73, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.118 INFO Extent 74 has flush number mismatch, : mend
Sep 22 23:22:14.118 INFO First source client ID for extent 74, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:74  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:74  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:74  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO find dest for source 1 for extent at index 74, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO Extent 75 has flush number mismatch, : mend
Sep 22 23:22:14.119 INFO First source client ID for extent 75, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:75  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:75  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:75  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO find dest for source 1 for extent at index 75, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO Extent 76 has flush number mismatch, : mend
Sep 22 23:22:14.119 INFO First source client ID for extent 76, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:76  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:76  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:76  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO find dest for source 1 for extent at index 76, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO Extent 77 has flush number mismatch, : mend
Sep 22 23:22:14.119 INFO First source client ID for extent 77, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:77  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:77  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:77  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO find dest for source 1 for extent at index 77, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO Extent 78 has flush number mismatch, : mend
Sep 22 23:22:14.119 INFO First source client ID for extent 78, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:78  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:78  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:78  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO find dest for source 1 for extent at index 78, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO Extent 79 has flush number mismatch, : mend
Sep 22 23:22:14.119 INFO First source client ID for extent 79, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:79  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:79  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:79  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO find dest for source 1 for extent at index 79, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO Extent 80 has flush number mismatch, : mend
Sep 22 23:22:14.119 INFO First source client ID for extent 80, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:80  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:80  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:80  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO find dest for source 1 for extent at index 80, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO Extent 81 has flush number mismatch, : mend
Sep 22 23:22:14.119 INFO First source client ID for extent 81, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:81  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:81  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:81  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO find dest for source 1 for extent at index 81, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO Extent 82 has flush number mismatch, : mend
Sep 22 23:22:14.119 INFO First source client ID for extent 82, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:82  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:82  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:82  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO find dest for source 1 for extent at index 82, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO Extent 83 has flush number mismatch, : mend
Sep 22 23:22:14.119 INFO First source client ID for extent 83, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:83  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:83  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:83  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO find dest for source 1 for extent at index 83, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO Extent 84 has flush number mismatch, : mend
Sep 22 23:22:14.119 INFO First source client ID for extent 84, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:84  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.119 INFO extent:84  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:84  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO find dest for source 1 for extent at index 84, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO Extent 85 has flush number mismatch, : mend
Sep 22 23:22:14.120 INFO First source client ID for extent 85, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:85  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:85  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:85  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO find dest for source 1 for extent at index 85, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO Extent 86 has flush number mismatch, : mend
Sep 22 23:22:14.120 INFO First source client ID for extent 86, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:86  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:86  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:86  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO find dest for source 1 for extent at index 86, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO Extent 87 has flush number mismatch, : mend
Sep 22 23:22:14.120 INFO First source client ID for extent 87, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:87  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:87  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:87  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO find dest for source 1 for extent at index 87, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO Extent 88 has flush number mismatch, : mend
Sep 22 23:22:14.120 INFO First source client ID for extent 88, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:88  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:88  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:88  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO find dest for source 1 for extent at index 88, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO Extent 89 has flush number mismatch, : mend
Sep 22 23:22:14.120 INFO First source client ID for extent 89, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:89  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:89  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:89  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO find dest for source 1 for extent at index 89, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO Extent 90 has flush number mismatch, : mend
Sep 22 23:22:14.120 INFO First source client ID for extent 90, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:90  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:90  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:90  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO find dest for source 1 for extent at index 90, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO Extent 91 has flush number mismatch, : mend
Sep 22 23:22:14.120 INFO First source client ID for extent 91, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:91  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:91  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:91  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO find dest for source 1 for extent at index 91, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO Extent 92 has flush number mismatch, : mend
Sep 22 23:22:14.120 INFO First source client ID for extent 92, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:92  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:92  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:92  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO find dest for source 1 for extent at index 92, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO Extent 93 has flush number mismatch, : mend
Sep 22 23:22:14.120 INFO First source client ID for extent 93, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:93  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:93  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:93  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO find dest for source 1 for extent at index 93, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO Extent 94 has flush number mismatch, : mend
Sep 22 23:22:14.120 INFO First source client ID for extent 94, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:94  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO extent:94  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.120 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:94  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO find dest for source 1 for extent at index 94, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO Extent 95 has flush number mismatch, : mend
Sep 22 23:22:14.121 INFO First source client ID for extent 95, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:95  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:95  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:95  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO find dest for source 1 for extent at index 95, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO Extent 96 has flush number mismatch, : mend
Sep 22 23:22:14.121 INFO First source client ID for extent 96, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:96  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:96  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:96  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO find dest for source 1 for extent at index 96, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO Extent 97 has flush number mismatch, : mend
Sep 22 23:22:14.121 INFO First source client ID for extent 97, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:97  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:97  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:97  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO find dest for source 1 for extent at index 97, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO Extent 98 has flush number mismatch, : mend
Sep 22 23:22:14.121 INFO First source client ID for extent 98, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:98  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:98  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:98  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO find dest for source 1 for extent at index 98, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO Extent 99 has flush number mismatch, : mend
Sep 22 23:22:14.121 INFO First source client ID for extent 99, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:99  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:99  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:99  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO find dest for source 1 for extent at index 99, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO Extent 100 has flush number mismatch, : mend
Sep 22 23:22:14.121 INFO First source client ID for extent 100, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:100  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:100  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:100  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO find dest for source 1 for extent at index 100, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO Extent 101 has flush number mismatch, : mend
Sep 22 23:22:14.121 INFO First source client ID for extent 101, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:101  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:101  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:101  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO find dest for source 1 for extent at index 101, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO Extent 102 has flush number mismatch, : mend
Sep 22 23:22:14.121 INFO First source client ID for extent 102, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:102  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:102  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:102  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO find dest for source 1 for extent at index 102, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO Extent 103 has flush number mismatch, : mend
Sep 22 23:22:14.121 INFO First source client ID for extent 103, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:103  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:103  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:103  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO find dest for source 1 for extent at index 103, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO Extent 104 has flush number mismatch, : mend
Sep 22 23:22:14.121 INFO First source client ID for extent 104, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:104  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:104  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.121 INFO extent:104  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO find dest for source 1 for extent at index 104, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO Extent 105 has flush number mismatch, : mend
Sep 22 23:22:14.122 INFO First source client ID for extent 105, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:105  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:105  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:105  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO find dest for source 1 for extent at index 105, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO Extent 106 has flush number mismatch, : mend
Sep 22 23:22:14.122 INFO First source client ID for extent 106, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:106  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:106  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:106  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO find dest for source 1 for extent at index 106, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO Extent 107 has flush number mismatch, : mend
Sep 22 23:22:14.122 INFO First source client ID for extent 107, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:107  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:107  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:107  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO find dest for source 1 for extent at index 107, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO Extent 108 has flush number mismatch, : mend
Sep 22 23:22:14.122 INFO First source client ID for extent 108, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:108  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:108  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:108  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO find dest for source 1 for extent at index 108, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO Extent 109 has flush number mismatch, : mend
Sep 22 23:22:14.122 INFO First source client ID for extent 109, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:109  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:109  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:109  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO find dest for source 1 for extent at index 109, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO Extent 110 has flush number mismatch, : mend
Sep 22 23:22:14.122 INFO First source client ID for extent 110, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:110  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:110  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:110  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO find dest for source 1 for extent at index 110, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO Extent 111 has flush number mismatch, : mend
Sep 22 23:22:14.122 INFO First source client ID for extent 111, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:111  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:111  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:111  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO find dest for source 1 for extent at index 111, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO Extent 112 has flush number mismatch, : mend
Sep 22 23:22:14.122 INFO First source client ID for extent 112, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:112  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:112  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:112  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO find dest for source 1 for extent at index 112, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO Extent 113 has flush number mismatch, : mend
Sep 22 23:22:14.122 INFO First source client ID for extent 113, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:113  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:113  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:113  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO find dest for source 1 for extent at index 113, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO Extent 114 has flush number mismatch, : mend
Sep 22 23:22:14.122 INFO First source client ID for extent 114, mrl: flush_mismatch, : mend
Sep 22 23:22:14.122 INFO extent:114  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:114  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:114  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO find dest for source 1 for extent at index 114, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO Extent 115 has flush number mismatch, : mend
Sep 22 23:22:14.123 INFO First source client ID for extent 115, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:115  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:115  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:115  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO find dest for source 1 for extent at index 115, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO Extent 116 has flush number mismatch, : mend
Sep 22 23:22:14.123 INFO First source client ID for extent 116, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:116  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:116  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:116  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO find dest for source 1 for extent at index 116, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO Extent 117 has flush number mismatch, : mend
Sep 22 23:22:14.123 INFO First source client ID for extent 117, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:117  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:117  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:117  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO find dest for source 1 for extent at index 117, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO Extent 118 has flush number mismatch, : mend
Sep 22 23:22:14.123 INFO First source client ID for extent 118, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:118  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:118  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:118  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO find dest for source 1 for extent at index 118, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO Extent 119 has flush number mismatch, : mend
Sep 22 23:22:14.123 INFO First source client ID for extent 119, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:119  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:119  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:119  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO find dest for source 1 for extent at index 119, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO Extent 120 has flush number mismatch, : mend
Sep 22 23:22:14.123 INFO First source client ID for extent 120, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:120  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:120  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:120  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO find dest for source 1 for extent at index 120, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO Extent 121 has flush number mismatch, : mend
Sep 22 23:22:14.123 INFO First source client ID for extent 121, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:121  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:121  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:121  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO find dest for source 1 for extent at index 121, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO Extent 122 has flush number mismatch, : mend
Sep 22 23:22:14.123 INFO First source client ID for extent 122, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:122  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:122  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:122  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO find dest for source 1 for extent at index 122, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO Extent 123 has flush number mismatch, : mend
Sep 22 23:22:14.123 INFO First source client ID for extent 123, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:123  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:123  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:123  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO find dest for source 1 for extent at index 123, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO Extent 124 has flush number mismatch, : mend
Sep 22 23:22:14.123 INFO First source client ID for extent 124, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:124  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.123 INFO extent:124  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:124  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO find dest for source 1 for extent at index 124, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO Extent 125 has flush number mismatch, : mend
Sep 22 23:22:14.124 INFO First source client ID for extent 125, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:125  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:125  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:125  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO find dest for source 1 for extent at index 125, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO Extent 126 has flush number mismatch, : mend
Sep 22 23:22:14.124 INFO First source client ID for extent 126, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:126  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:126  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:126  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO find dest for source 1 for extent at index 126, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO Extent 127 has flush number mismatch, : mend
Sep 22 23:22:14.124 INFO First source client ID for extent 127, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:127  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:127  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:127  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO find dest for source 1 for extent at index 127, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO Extent 128 has flush number mismatch, : mend
Sep 22 23:22:14.124 INFO First source client ID for extent 128, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:128  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:128  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:128  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO find dest for source 1 for extent at index 128, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO Extent 129 has flush number mismatch, : mend
Sep 22 23:22:14.124 INFO First source client ID for extent 129, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:129  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:129  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:129  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO find dest for source 1 for extent at index 129, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO Extent 130 has flush number mismatch, : mend
Sep 22 23:22:14.124 INFO First source client ID for extent 130, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:130  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:130  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:130  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO find dest for source 1 for extent at index 130, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO Extent 131 has flush number mismatch, : mend
Sep 22 23:22:14.124 INFO First source client ID for extent 131, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:131  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:131  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:131  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO find dest for source 1 for extent at index 131, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO Extent 132 has flush number mismatch, : mend
Sep 22 23:22:14.124 INFO First source client ID for extent 132, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:132  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:132  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:132  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO find dest for source 1 for extent at index 132, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO Extent 133 has flush number mismatch, : mend
Sep 22 23:22:14.124 INFO First source client ID for extent 133, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:133  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:133  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:133  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO find dest for source 1 for extent at index 133, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO Extent 134 has flush number mismatch, : mend
Sep 22 23:22:14.124 INFO First source client ID for extent 134, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:134  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.124 INFO extent:134  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:134  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO find dest for source 1 for extent at index 134, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO Extent 135 has flush number mismatch, : mend
Sep 22 23:22:14.125 INFO First source client ID for extent 135, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:135  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:135  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:135  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO find dest for source 1 for extent at index 135, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO Extent 136 has flush number mismatch, : mend
Sep 22 23:22:14.125 INFO First source client ID for extent 136, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:136  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:136  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:136  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO find dest for source 1 for extent at index 136, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO Extent 137 has flush number mismatch, : mend
Sep 22 23:22:14.125 INFO First source client ID for extent 137, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:137  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:137  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:137  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO find dest for source 1 for extent at index 137, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO Extent 138 has flush number mismatch, : mend
Sep 22 23:22:14.125 INFO First source client ID for extent 138, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:138  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:138  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:138  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO find dest for source 1 for extent at index 138, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO Extent 139 has flush number mismatch, : mend
Sep 22 23:22:14.125 INFO First source client ID for extent 139, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:139  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:139  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:139  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO find dest for source 1 for extent at index 139, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO Extent 140 has flush number mismatch, : mend
Sep 22 23:22:14.125 INFO First source client ID for extent 140, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:140  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:140  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:140  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO find dest for source 1 for extent at index 140, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO Extent 141 has flush number mismatch, : mend
Sep 22 23:22:14.125 INFO First source client ID for extent 141, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:141  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:141  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:141  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO find dest for source 1 for extent at index 141, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO Extent 142 has flush number mismatch, : mend
Sep 22 23:22:14.125 INFO First source client ID for extent 142, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:142  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:142  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:142  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO find dest for source 1 for extent at index 142, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO Extent 143 has flush number mismatch, : mend
Sep 22 23:22:14.125 INFO First source client ID for extent 143, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:143  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:143  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:143  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO find dest for source 1 for extent at index 143, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO Extent 144 has flush number mismatch, : mend
Sep 22 23:22:14.125 INFO First source client ID for extent 144, mrl: flush_mismatch, : mend
Sep 22 23:22:14.125 INFO extent:144  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:144  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:144  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO find dest for source 1 for extent at index 144, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO Extent 145 has flush number mismatch, : mend
Sep 22 23:22:14.126 INFO First source client ID for extent 145, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:145  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:145  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:145  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO find dest for source 1 for extent at index 145, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO Extent 146 has flush number mismatch, : mend
Sep 22 23:22:14.126 INFO First source client ID for extent 146, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:146  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:146  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:146  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO find dest for source 1 for extent at index 146, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO Extent 147 has flush number mismatch, : mend
Sep 22 23:22:14.126 INFO First source client ID for extent 147, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:147  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:147  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:147  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO find dest for source 1 for extent at index 147, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO Extent 148 has flush number mismatch, : mend
Sep 22 23:22:14.126 INFO First source client ID for extent 148, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:148  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:148  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:148  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO find dest for source 1 for extent at index 148, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO Extent 149 has flush number mismatch, : mend
Sep 22 23:22:14.126 INFO First source client ID for extent 149, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:149  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:149  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:149  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO find dest for source 1 for extent at index 149, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO Extent 150 has flush number mismatch, : mend
Sep 22 23:22:14.126 INFO First source client ID for extent 150, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:150  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:150  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:150  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO find dest for source 1 for extent at index 150, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO Extent 151 has flush number mismatch, : mend
Sep 22 23:22:14.126 INFO First source client ID for extent 151, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:151  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:151  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:151  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO find dest for source 1 for extent at index 151, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO Extent 152 has flush number mismatch, : mend
Sep 22 23:22:14.126 INFO First source client ID for extent 152, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:152  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:152  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:152  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO find dest for source 1 for extent at index 152, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO Extent 153 has flush number mismatch, : mend
Sep 22 23:22:14.126 INFO First source client ID for extent 153, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:153  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:153  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:153  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO find dest for source 1 for extent at index 153, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO Extent 154 has flush number mismatch, : mend
Sep 22 23:22:14.126 INFO First source client ID for extent 154, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:154  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.126 INFO extent:154  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:154  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO find dest for source 1 for extent at index 154, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO Extent 155 has flush number mismatch, : mend
Sep 22 23:22:14.127 INFO First source client ID for extent 155, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:155  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:155  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:155  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO find dest for source 1 for extent at index 155, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO Extent 156 has flush number mismatch, : mend
Sep 22 23:22:14.127 INFO First source client ID for extent 156, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:156  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:156  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:156  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO find dest for source 1 for extent at index 156, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO Extent 157 has flush number mismatch, : mend
Sep 22 23:22:14.127 INFO First source client ID for extent 157, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:157  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:157  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:157  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO find dest for source 1 for extent at index 157, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO Extent 158 has flush number mismatch, : mend
Sep 22 23:22:14.127 INFO First source client ID for extent 158, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:158  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:158  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:158  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO find dest for source 1 for extent at index 158, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO Extent 159 has flush number mismatch, : mend
Sep 22 23:22:14.127 INFO First source client ID for extent 159, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:159  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:159  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:159  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO find dest for source 1 for extent at index 159, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO Extent 160 has flush number mismatch, : mend
Sep 22 23:22:14.127 INFO First source client ID for extent 160, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:160  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:160  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:160  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO find dest for source 1 for extent at index 160, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO Extent 161 has flush number mismatch, : mend
Sep 22 23:22:14.127 INFO First source client ID for extent 161, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:161  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:161  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:161  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO find dest for source 1 for extent at index 161, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO Extent 162 has flush number mismatch, : mend
Sep 22 23:22:14.127 INFO First source client ID for extent 162, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:162  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:162  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:162  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO find dest for source 1 for extent at index 162, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO Extent 163 has flush number mismatch, : mend
Sep 22 23:22:14.127 INFO First source client ID for extent 163, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:163  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:163  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:163  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO find dest for source 1 for extent at index 163, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO Extent 164 has flush number mismatch, : mend
Sep 22 23:22:14.127 INFO First source client ID for extent 164, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:164  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO extent:164  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.127 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:164  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO find dest for source 1 for extent at index 164, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO Extent 165 has flush number mismatch, : mend
Sep 22 23:22:14.128 INFO First source client ID for extent 165, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:165  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:165  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:165  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO find dest for source 1 for extent at index 165, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO Extent 166 has flush number mismatch, : mend
Sep 22 23:22:14.128 INFO First source client ID for extent 166, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:166  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:166  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:166  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO find dest for source 1 for extent at index 166, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO Extent 167 has flush number mismatch, : mend
Sep 22 23:22:14.128 INFO First source client ID for extent 167, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:167  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:167  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:167  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO find dest for source 1 for extent at index 167, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO Extent 168 has flush number mismatch, : mend
Sep 22 23:22:14.128 INFO First source client ID for extent 168, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:168  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:168  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:168  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO find dest for source 1 for extent at index 168, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO Extent 169 has flush number mismatch, : mend
Sep 22 23:22:14.128 INFO First source client ID for extent 169, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:169  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:169  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:169  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO find dest for source 1 for extent at index 169, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO Extent 170 has flush number mismatch, : mend
Sep 22 23:22:14.128 INFO First source client ID for extent 170, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:170  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:170  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:170  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO find dest for source 1 for extent at index 170, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO Extent 171 has flush number mismatch, : mend
Sep 22 23:22:14.128 INFO First source client ID for extent 171, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:171  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:171  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:171  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO find dest for source 1 for extent at index 171, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO Extent 172 has flush number mismatch, : mend
Sep 22 23:22:14.128 INFO First source client ID for extent 172, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:172  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:172  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:172  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO find dest for source 1 for extent at index 172, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO Extent 173 has flush number mismatch, : mend
Sep 22 23:22:14.128 INFO First source client ID for extent 173, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:173  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:173  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:173  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO find dest for source 1 for extent at index 173, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO Extent 174 has flush number mismatch, : mend
Sep 22 23:22:14.128 INFO First source client ID for extent 174, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:174  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:174  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.128 INFO extent:174  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO find dest for source 1 for extent at index 174, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO Extent 175 has flush number mismatch, : mend
Sep 22 23:22:14.129 INFO First source client ID for extent 175, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:175  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:175  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:175  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO find dest for source 1 for extent at index 175, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO Extent 176 has flush number mismatch, : mend
Sep 22 23:22:14.129 INFO First source client ID for extent 176, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:176  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:176  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:176  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO find dest for source 1 for extent at index 176, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO Extent 177 has flush number mismatch, : mend
Sep 22 23:22:14.129 INFO First source client ID for extent 177, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:177  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:177  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:177  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO find dest for source 1 for extent at index 177, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO Extent 178 has flush number mismatch, : mend
Sep 22 23:22:14.129 INFO First source client ID for extent 178, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:178  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:178  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:178  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO find dest for source 1 for extent at index 178, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO Extent 179 has flush number mismatch, : mend
Sep 22 23:22:14.129 INFO First source client ID for extent 179, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:179  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:179  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:179  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO find dest for source 1 for extent at index 179, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO Extent 180 has flush number mismatch, : mend
Sep 22 23:22:14.129 INFO First source client ID for extent 180, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:180  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:180  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:180  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO find dest for source 1 for extent at index 180, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO Extent 181 has flush number mismatch, : mend
Sep 22 23:22:14.129 INFO First source client ID for extent 181, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:181  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:181  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:181  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO find dest for source 1 for extent at index 181, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO Extent 182 has flush number mismatch, : mend
Sep 22 23:22:14.129 INFO First source client ID for extent 182, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:182  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:182  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:182  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO find dest for source 1 for extent at index 182, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO Extent 183 has flush number mismatch, : mend
Sep 22 23:22:14.129 INFO First source client ID for extent 183, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:183  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:183  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:183  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO find dest for source 1 for extent at index 183, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO Extent 184 has flush number mismatch, : mend
Sep 22 23:22:14.129 INFO First source client ID for extent 184, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:184  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:184  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO extent:184  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.129 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO find dest for source 1 for extent at index 184, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO Extent 185 has flush number mismatch, : mend
Sep 22 23:22:14.130 INFO First source client ID for extent 185, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO extent:185  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO extent:185  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO extent:185  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO find dest for source 1 for extent at index 185, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO Extent 186 has flush number mismatch, : mend
Sep 22 23:22:14.130 INFO First source client ID for extent 186, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO extent:186  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO extent:186  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO extent:186  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO find dest for source 1 for extent at index 186, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO Extent 187 has flush number mismatch, : mend
Sep 22 23:22:14.130 INFO First source client ID for extent 187, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO extent:187  gens: 0 1 1, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO extent:187  flush: 0 1 1 scs: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO max_flush now has: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO extent:187  dirty: false false false, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO No maxes found, left with: [ClientId(1), ClientId(2)], mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO find dest for source 1 for extent at index 187, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO source 1, add dest 0 gen, mrl: flush_mismatch, : mend
Sep 22 23:22:14.130 INFO Transition from WaitQuorum to Repair
Sep 22 23:22:14.130 INFO Transition from WaitQuorum to Repair
Sep 22 23:22:14.130 INFO Transition from WaitQuorum to Repair
Sep 22 23:22:14.130 INFO Found 188 extents that need repair
Sep 22 23:22:14.130 INFO Full repair list: {31: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 175: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 3: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 53: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 59: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 103: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 92: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 129: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 132: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 138: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 147: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 67: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 2: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 25: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 152: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 153: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 156: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 125: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 37: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 150: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 157: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 89: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 163: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 73: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 168: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 176: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 38: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 21: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 97: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 124: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 47: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 179: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 19: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 167: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 182: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 78: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 52: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 172: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 183: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 11: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 184: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 41: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 77: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 107: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 64: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 171: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 165: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 118: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 111: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 74: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 117: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 75: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 36: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 68: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 123: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 55: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 159: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 35: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 162: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 121: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 65: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 106: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 22: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 146: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 148: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 6: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 62: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 126: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 128: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 79: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 144: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 160: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 133: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 109: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 169: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 28: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 49: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 15: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 43: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 60: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 18: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 110: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 136: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 61: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 119: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 30: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 46: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 127: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 99: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 12: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 34: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 83: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 90: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 66: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 87: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 139: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 26: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 84: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 56: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 94: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 93: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 44: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 45: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 115: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 32: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 141: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 9: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 186: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 130: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 122: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 105: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 137: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 7: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 63: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 58: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 96: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 40: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 80: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 149: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 161: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 71: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 5: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 23: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 98: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 1: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 166: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 185: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 86: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 135: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 13: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 29: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 27: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 187: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 10: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 102: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 85: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 100: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 95: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 154: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 50: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 91: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 158: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 177: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 134: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 69: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 155: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 0: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 164: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 24: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 39: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 48: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 16: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 4: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 70: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 76: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 113: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 108: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 42: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 140: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 178: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 181: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 82: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 174: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 17: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 116: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 142: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 88: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 51: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 33: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 54: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 112: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 143: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 151: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 101: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 173: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 20: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 104: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 81: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 120: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 145: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 131: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 170: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 72: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 180: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 57: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 14: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 8: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }, 114: ExtentFix { source: ClientId(1), dest: [ClientId(0)] }}, : downstairs
Sep 22 23:22:14.131 INFO Task list: [ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 31 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 31 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 175 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 175 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 3 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 3 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 53 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 53 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 59 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 59 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 103 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 103 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 92 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 92 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 129 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 129 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 132 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 132 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 138 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 138 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 147 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 147 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 67 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 67 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(52), op: ExtentFlush { repair_id: ReconciliationId(52), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(53), op: ExtentClose { repair_id: ReconciliationId(53), extent_id: 25 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(54), op: ExtentRepair { repair_id: ReconciliationId(54), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(55), op: ExtentReopen { repair_id: ReconciliationId(55), extent_id: 25 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(56), op: ExtentFlush { repair_id: ReconciliationId(56), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(57), op: ExtentClose { repair_id: ReconciliationId(57), extent_id: 152 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(58), op: ExtentRepair { repair_id: ReconciliationId(58), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(59), op: ExtentReopen { repair_id: ReconciliationId(59), extent_id: 152 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(60), op: ExtentFlush { repair_id: ReconciliationId(60), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(61), op: ExtentClose { repair_id: ReconciliationId(61), extent_id: 153 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(62), op: ExtentRepair { repair_id: ReconciliationId(62), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(63), op: ExtentReopen { repair_id: ReconciliationId(63), extent_id: 153 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(64), op: ExtentFlush { repair_id: ReconciliationId(64), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(65), op: ExtentClose { repair_id: ReconciliationId(65), extent_id: 156 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(66), op: ExtentRepair { repair_id: ReconciliationId(66), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(67), op: ExtentReopen { repair_id: ReconciliationId(67), extent_id: 156 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(68), op: ExtentFlush { repair_id: ReconciliationId(68), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(69), op: ExtentClose { repair_id: ReconciliationId(69), extent_id: 125 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(70), op: ExtentRepair { repair_id: ReconciliationId(70), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(71), op: ExtentReopen { repair_id: ReconciliationId(71), extent_id: 125 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(72), op: ExtentFlush { repair_id: ReconciliationId(72), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(73), op: ExtentClose { repair_id: ReconciliationId(73), extent_id: 37 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(74), op: ExtentRepair { repair_id: ReconciliationId(74), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(75), op: ExtentReopen { repair_id: ReconciliationId(75), extent_id: 37 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(76), op: ExtentFlush { repair_id: ReconciliationId(76), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(77), op: ExtentClose { repair_id: ReconciliationId(77), extent_id: 150 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(78), op: ExtentRepair { repair_id: ReconciliationId(78), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(79), op: ExtentReopen { repair_id: ReconciliationId(79), extent_id: 150 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(80), op: ExtentFlush { repair_id: ReconciliationId(80), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(81), op: ExtentClose { repair_id: ReconciliationId(81), extent_id: 157 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(82), op: ExtentRepair { repair_id: ReconciliationId(82), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(83), op: ExtentReopen { repair_id: ReconciliationId(83), extent_id: 157 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(84), op: ExtentFlush { repair_id: ReconciliationId(84), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(85), op: ExtentClose { repair_id: ReconciliationId(85), extent_id: 89 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(86), op: ExtentRepair { repair_id: ReconciliationId(86), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(87), op: ExtentReopen { repair_id: ReconciliationId(87), extent_id: 89 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(88), op: ExtentFlush { repair_id: ReconciliationId(88), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(89), op: ExtentClose { repair_id: ReconciliationId(89), extent_id: 163 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(90), op: ExtentRepair { repair_id: ReconciliationId(90), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(91), op: ExtentReopen { repair_id: ReconciliationId(91), extent_id: 163 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(92), op: ExtentFlush { repair_id: ReconciliationId(92), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(93), op: ExtentClose { repair_id: ReconciliationId(93), extent_id: 73 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(94), op: ExtentRepair { repair_id: ReconciliationId(94), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(95), op: ExtentReopen { repair_id: ReconciliationId(95), extent_id: 73 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(96), op: ExtentFlush { repair_id: ReconciliationId(96), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(97), op: ExtentClose { repair_id: ReconciliationId(97), extent_id: 168 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(98), op: ExtentRepair { repair_id: ReconciliationId(98), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(99), op: ExtentReopen { repair_id: ReconciliationId(99), extent_id: 168 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(100), op: ExtentFlush { repair_id: ReconciliationId(100), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(101), op: ExtentClose { repair_id: ReconciliationId(101), extent_id: 176 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(102), op: ExtentRepair { repair_id: ReconciliationId(102), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(103), op: ExtentReopen { repair_id: ReconciliationId(103), extent_id: 176 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(104), op: ExtentFlush { repair_id: ReconciliationId(104), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(105), op: ExtentClose { repair_id: ReconciliationId(105), extent_id: 38 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(106), op: ExtentRepair { repair_id: ReconciliationId(106), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(107), op: ExtentReopen { repair_id: ReconciliationId(107), extent_id: 38 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(108), op: ExtentFlush { repair_id: ReconciliationId(108), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(109), op: ExtentClose { repair_id: ReconciliationId(109), extent_id: 21 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(110), op: ExtentRepair { repair_id: ReconciliationId(110), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(111), op: ExtentReopen { repair_id: ReconciliationId(111), extent_id: 21 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(112), op: ExtentFlush { repair_id: ReconciliationId(112), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(113), op: ExtentClose { repair_id: ReconciliationId(113), extent_id: 97 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(114), op: ExtentRepair { repair_id: ReconciliationId(114), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(115), op: ExtentReopen { repair_id: ReconciliationId(115), extent_id: 97 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(116), op: ExtentFlush { repair_id: ReconciliationId(116), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(117), op: ExtentClose { repair_id: ReconciliationId(117), extent_id: 124 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(118), op: ExtentRepair { repair_id: ReconciliationId(118), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(119), op: ExtentReopen { repair_id: ReconciliationId(119), extent_id: 124 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(120), op: ExtentFlush { repair_id: ReconciliationId(120), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(121), op: ExtentClose { repair_id: ReconciliationId(121), extent_id: 47 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(122), op: ExtentRepair { repair_id: ReconciliationId(122), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(123), op: ExtentReopen { repair_id: ReconciliationId(123), extent_id: 47 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(124), op: ExtentFlush { repair_id: ReconciliationId(124), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(125), op: ExtentClose { repair_id: ReconciliationId(125), extent_id: 179 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(126), op: ExtentRepair { repair_id: ReconciliationId(126), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(127), op: ExtentReopen { repair_id: ReconciliationId(127), extent_id: 179 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(128), op: ExtentFlush { repair_id: ReconciliationId(128), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(129), op: ExtentClose { repair_id: ReconciliationId(129), extent_id: 19 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(130), op: ExtentRepair { repair_id: ReconciliationId(130), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(131), op: ExtentReopen { repair_id: ReconciliationId(131), extent_id: 19 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(132), op: ExtentFlush { repair_id: ReconciliationId(132), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(133), op: ExtentClose { repair_id: ReconciliationId(133), extent_id: 167 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(134), op: ExtentRepair { repair_id: ReconciliationId(134), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(135), op: ExtentReopen { repair_id: ReconciliationId(135), extent_id: 167 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(136), op: ExtentFlush { repair_id: ReconciliationId(136), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(137), op: ExtentClose { repair_id: ReconciliationId(137), extent_id: 182 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(138), op: ExtentRepair { repair_id: ReconciliationId(138), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(139), op: ExtentReopen { repair_id: ReconciliationId(139), extent_id: 182 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(140), op: ExtentFlush { repair_id: ReconciliationId(140), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(141), op: ExtentClose { repair_id: ReconciliationId(141), extent_id: 78 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(142), op: ExtentRepair { repair_id: ReconciliationId(142), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(143), op: ExtentReopen { repair_id: ReconciliationId(143), extent_id: 78 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(144), op: ExtentFlush { repair_id: ReconciliationId(144), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(145), op: ExtentClose { repair_id: ReconciliationId(145), extent_id: 52 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(146), op: ExtentRepair { repair_id: ReconciliationId(146), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(147), op: ExtentReopen { repair_id: ReconciliationId(147), extent_id: 52 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(148), op: ExtentFlush { repair_id: ReconciliationId(148), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(149), op: ExtentClose { repair_id: ReconciliationId(149), extent_id: 172 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(150), op: ExtentRepair { repair_id: ReconciliationId(150), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(151), op: ExtentReopen { repair_id: ReconciliationId(151), extent_id: 172 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(152), op: ExtentFlush { repair_id: ReconciliationId(152), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(153), op: ExtentClose { repair_id: ReconciliationId(153), extent_id: 183 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(154), op: ExtentRepair { repair_id: ReconciliationId(154), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(155), op: ExtentReopen { repair_id: ReconciliationId(155), extent_id: 183 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(156), op: ExtentFlush { repair_id: ReconciliationId(156), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(157), op: ExtentClose { repair_id: ReconciliationId(157), extent_id: 11 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(158), op: ExtentRepair { repair_id: ReconciliationId(158), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(159), op: ExtentReopen { repair_id: ReconciliationId(159), extent_id: 11 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(160), op: ExtentFlush { repair_id: ReconciliationId(160), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(161), op: ExtentClose { repair_id: ReconciliationId(161), extent_id: 184 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(162), op: ExtentRepair { repair_id: ReconciliationId(162), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(163), op: ExtentReopen { repair_id: ReconciliationId(163), extent_id: 184 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(164), op: ExtentFlush { repair_id: ReconciliationId(164), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(165), op: ExtentClose { repair_id: ReconciliationId(165), extent_id: 41 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(166), op: ExtentRepair { repair_id: ReconciliationId(166), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(167), op: ExtentReopen { repair_id: ReconciliationId(167), extent_id: 41 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(168), op: ExtentFlush { repair_id: ReconciliationId(168), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(169), op: ExtentClose { repair_id: ReconciliationId(169), extent_id: 77 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(170), op: ExtentRepair { repair_id: ReconciliationId(170), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(171), op: ExtentReopen { repair_id: ReconciliationId(171), extent_id: 77 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(172), op: ExtentFlush { repair_id: ReconciliationId(172), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(173), op: ExtentClose { repair_id: ReconciliationId(173), extent_id: 107 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(174), op: ExtentRepair { repair_id: ReconciliationId(174), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(175), op: ExtentReopen { repair_id: ReconciliationId(175), extent_id: 107 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(176), op: ExtentFlush { repair_id: ReconciliationId(176), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(177), op: ExtentClose { repair_id: ReconciliationId(177), extent_id: 64 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(178), op: ExtentRepair { repair_id: ReconciliationId(178), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(179), op: ExtentReopen { repair_id: ReconciliationId(179), extent_id: 64 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(180), op: ExtentFlush { repair_id: ReconciliationId(180), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(181), op: ExtentClose { repair_id: ReconciliationId(181), extent_id: 171 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(182), op: ExtentRepair { repair_id: ReconciliationId(182), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(183), op: ExtentReopen { repair_id: ReconciliationId(183), extent_id: 171 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(184), op: ExtentFlush { repair_id: ReconciliationId(184), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(185), op: ExtentClose { repair_id: ReconciliationId(185), extent_id: 165 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(186), op: ExtentRepair { repair_id: ReconciliationId(186), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(187), op: ExtentReopen { repair_id: ReconciliationId(187), extent_id: 165 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(188), op: ExtentFlush { repair_id: ReconciliationId(188), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(189), op: ExtentClose { repair_id: ReconciliationId(189), extent_id: 118 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(190), op: ExtentRepair { repair_id: ReconciliationId(190), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(191), op: ExtentReopen { repair_id: ReconciliationId(191), extent_id: 118 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(192), op: ExtentFlush { repair_id: ReconciliationId(192), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(193), op: ExtentClose { repair_id: ReconciliationId(193), extent_id: 111 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(194), op: ExtentRepair { repair_id: ReconciliationId(194), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(195), op: ExtentReopen { repair_id: ReconciliationId(195), extent_id: 111 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(196), op: ExtentFlush { repair_id: ReconciliationId(196), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(197), op: ExtentClose { repair_id: ReconciliationId(197), extent_id: 74 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(198), op: ExtentRepair { repair_id: ReconciliationId(198), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(199), op: ExtentReopen { repair_id: ReconciliationId(199), extent_id: 74 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(200), op: ExtentFlush { repair_id: ReconciliationId(200), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(201), op: ExtentClose { repair_id: ReconciliationId(201), extent_id: 117 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(202), op: ExtentRepair { repair_id: ReconciliationId(202), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(203), op: ExtentReopen { repair_id: ReconciliationId(203), extent_id: 117 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(204), op: ExtentFlush { repair_id: ReconciliationId(204), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(205), op: ExtentClose { repair_id: ReconciliationId(205), extent_id: 75 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(206), op: ExtentRepair { repair_id: ReconciliationId(206), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(207), op: ExtentReopen { repair_id: ReconciliationId(207), extent_id: 75 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(208), op: ExtentFlush { repair_id: ReconciliationId(208), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(209), op: ExtentClose { repair_id: ReconciliationId(209), extent_id: 36 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(210), op: ExtentRepair { repair_id: ReconciliationId(210), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(211), op: ExtentReopen { repair_id: ReconciliationId(211), extent_id: 36 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(212), op: ExtentFlush { repair_id: ReconciliationId(212), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(213), op: ExtentClose { repair_id: ReconciliationId(213), extent_id: 68 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(214), op: ExtentRepair { repair_id: ReconciliationId(214), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(215), op: ExtentReopen { repair_id: ReconciliationId(215), extent_id: 68 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(216), op: ExtentFlush { repair_id: ReconciliationId(216), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(217), op: ExtentClose { repair_id: ReconciliationId(217), extent_id: 123 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(218), op: ExtentRepair { repair_id: ReconciliationId(218), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(219), op: ExtentReopen { repair_id: ReconciliationId(219), extent_id: 123 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(220), op: ExtentFlush { repair_id: ReconciliationId(220), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(221), op: ExtentClose { repair_id: ReconciliationId(221), extent_id: 55 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(222), op: ExtentRepair { repair_id: ReconciliationId(222), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(223), op: ExtentReopen { repair_id: ReconciliationId(223), extent_id: 55 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(224), op: ExtentFlush { repair_id: ReconciliationId(224), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(225), op: ExtentClose { repair_id: ReconciliationId(225), extent_id: 159 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(226), op: ExtentRepair { repair_id: ReconciliationId(226), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(227), op: ExtentReopen { repair_id: ReconciliationId(227), extent_id: 159 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(228), op: ExtentFlush { repair_id: ReconciliationId(228), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(229), op: ExtentClose { repair_id: ReconciliationId(229), extent_id: 35 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(230), op: ExtentRepair { repair_id: ReconciliationId(230), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(231), op: ExtentReopen { repair_id: ReconciliationId(231), extent_id: 35 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(232), op: ExtentFlush { repair_id: ReconciliationId(232), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(233), op: ExtentClose { repair_id: ReconciliationId(233), extent_id: 162 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(234), op: ExtentRepair { repair_id: ReconciliationId(234), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(235), op: ExtentReopen { repair_id: ReconciliationId(235), extent_id: 162 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(236), op: ExtentFlush { repair_id: ReconciliationId(236), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(237), op: ExtentClose { repair_id: ReconciliationId(237), extent_id: 121 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(238), op: ExtentRepair { repair_id: ReconciliationId(238), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(239), op: ExtentReopen { repair_id: ReconciliationId(239), extent_id: 121 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(240), op: ExtentFlush { repair_id: ReconciliationId(240), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(241), op: ExtentClose { repair_id: ReconciliationId(241), extent_id: 65 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(242), op: ExtentRepair { repair_id: ReconciliationId(242), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(243), op: ExtentReopen { repair_id: ReconciliationId(243), extent_id: 65 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(244), op: ExtentFlush { repair_id: ReconciliationId(244), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(245), op: ExtentClose { repair_id: ReconciliationId(245), extent_id: 106 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(246), op: ExtentRepair { repair_id: ReconciliationId(246), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(247), op: ExtentReopen { repair_id: ReconciliationId(247), extent_id: 106 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(248), op: ExtentFlush { repair_id: ReconciliationId(248), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(249), op: ExtentClose { repair_id: ReconciliationId(249), extent_id: 22 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(250), op: ExtentRepair { repair_id: ReconciliationId(250), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(251), op: ExtentReopen { repair_id: ReconciliationId(251), extent_id: 22 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(252), op: ExtentFlush { repair_id: ReconciliationId(252), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(253), op: ExtentClose { repair_id: ReconciliationId(253), extent_id: 146 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(254), op: ExtentRepair { repair_id: ReconciliationId(254), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(255), op: ExtentReopen { repair_id: ReconciliationId(255), extent_id: 146 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(256), op: ExtentFlush { repair_id: ReconciliationId(256), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(257), op: ExtentClose { repair_id: ReconciliationId(257), extent_id: 148 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(258), op: ExtentRepair { repair_id: ReconciliationId(258), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(259), op: ExtentReopen { repair_id: ReconciliationId(259), extent_id: 148 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(260), op: ExtentFlush { repair_id: ReconciliationId(260), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(261), op: ExtentClose { repair_id: ReconciliationId(261), extent_id: 6 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(262), op: ExtentRepair { repair_id: ReconciliationId(262), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(263), op: ExtentReopen { repair_id: ReconciliationId(263), extent_id: 6 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(264), op: ExtentFlush { repair_id: ReconciliationId(264), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(265), op: ExtentClose { repair_id: ReconciliationId(265), extent_id: 62 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(266), op: ExtentRepair { repair_id: ReconciliationId(266), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(267), op: ExtentReopen { repair_id: ReconciliationId(267), extent_id: 62 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(268), op: ExtentFlush { repair_id: ReconciliationId(268), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(269), op: ExtentClose { repair_id: ReconciliationId(269), extent_id: 126 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(270), op: ExtentRepair { repair_id: ReconciliationId(270), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(271), op: ExtentReopen { repair_id: ReconciliationId(271), extent_id: 126 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(272), op: ExtentFlush { repair_id: ReconciliationId(272), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(273), op: ExtentClose { repair_id: ReconciliationId(273), extent_id: 128 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(274), op: ExtentRepair { repair_id: ReconciliationId(274), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(275), op: ExtentReopen { repair_id: ReconciliationId(275), extent_id: 128 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(276), op: ExtentFlush { repair_id: ReconciliationId(276), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(277), op: ExtentClose { repair_id: ReconciliationId(277), extent_id: 79 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(278), op: ExtentRepair { repair_id: ReconciliationId(278), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(279), op: ExtentReopen { repair_id: ReconciliationId(279), extent_id: 79 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(280), op: ExtentFlush { repair_id: ReconciliationId(280), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(281), op: ExtentClose { repair_id: ReconciliationId(281), extent_id: 144 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(282), op: ExtentRepair { repair_id: ReconciliationId(282), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(283), op: ExtentReopen { repair_id: ReconciliationId(283), extent_id: 144 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(284), op: ExtentFlush { repair_id: ReconciliationId(284), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(285), op: ExtentClose { repair_id: ReconciliationId(285), extent_id: 160 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(286), op: ExtentRepair { repair_id: ReconciliationId(286), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(287), op: ExtentReopen { repair_id: ReconciliationId(287), extent_id: 160 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(288), op: ExtentFlush { repair_id: ReconciliationId(288), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(289), op: ExtentClose { repair_id: ReconciliationId(289), extent_id: 133 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(290), op: ExtentRepair { repair_id: ReconciliationId(290), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(291), op: ExtentReopen { repair_id: ReconciliationId(291), extent_id: 133 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(292), op: ExtentFlush { repair_id: ReconciliationId(292), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(293), op: ExtentClose { repair_id: ReconciliationId(293), extent_id: 109 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(294), op: ExtentRepair { repair_id: ReconciliationId(294), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(295), op: ExtentReopen { repair_id: ReconciliationId(295), extent_id: 109 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(296), op: ExtentFlush { repair_id: ReconciliationId(296), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(297), op: ExtentClose { repair_id: ReconciliationId(297), extent_id: 169 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(298), op: ExtentRepair { repair_id: ReconciliationId(298), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(299), op: ExtentReopen { repair_id: ReconciliationId(299), extent_id: 169 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(300), op: ExtentFlush { repair_id: ReconciliationId(300), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(301), op: ExtentClose { repair_id: ReconciliationId(301), extent_id: 28 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(302), op: ExtentRepair { repair_id: ReconciliationId(302), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(303), op: ExtentReopen { repair_id: ReconciliationId(303), extent_id: 28 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(304), op: ExtentFlush { repair_id: ReconciliationId(304), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(305), op: ExtentClose { repair_id: ReconciliationId(305), extent_id: 49 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(306), op: ExtentRepair { repair_id: ReconciliationId(306), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(307), op: ExtentReopen { repair_id: ReconciliationId(307), extent_id: 49 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(308), op: ExtentFlush { repair_id: ReconciliationId(308), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(309), op: ExtentClose { repair_id: ReconciliationId(309), extent_id: 15 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(310), op: ExtentRepair { repair_id: ReconciliationId(310), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(311), op: ExtentReopen { repair_id: ReconciliationId(311), extent_id: 15 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(312), op: ExtentFlush { repair_id: ReconciliationId(312), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(313), op: ExtentClose { repair_id: ReconciliationId(313), extent_id: 43 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(314), op: ExtentRepair { repair_id: ReconciliationId(314), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(315), op: ExtentReopen { repair_id: ReconciliationId(315), extent_id: 43 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(316), op: ExtentFlush { repair_id: ReconciliationId(316), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(317), op: ExtentClose { repair_id: ReconciliationId(317), extent_id: 60 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(318), op: ExtentRepair { repair_id: ReconciliationId(318), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(319), op: ExtentReopen { repair_id: ReconciliationId(319), extent_id: 60 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(320), op: ExtentFlush { repair_id: ReconciliationId(320), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(321), op: ExtentClose { repair_id: ReconciliationId(321), extent_id: 18 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(322), op: ExtentRepair { repair_id: ReconciliationId(322), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(323), op: ExtentReopen { repair_id: ReconciliationId(323), extent_id: 18 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(324), op: ExtentFlush { repair_id: ReconciliationId(324), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(325), op: ExtentClose { repair_id: ReconciliationId(325), extent_id: 110 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(326), op: ExtentRepair { repair_id: ReconciliationId(326), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(327), op: ExtentReopen { repair_id: ReconciliationId(327), extent_id: 110 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(328), op: ExtentFlush { repair_id: ReconciliationId(328), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(329), op: ExtentClose { repair_id: ReconciliationId(329), extent_id: 136 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(330), op: ExtentRepair { repair_id: ReconciliationId(330), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(331), op: ExtentReopen { repair_id: ReconciliationId(331), extent_id: 136 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(332), op: ExtentFlush { repair_id: ReconciliationId(332), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(333), op: ExtentClose { repair_id: ReconciliationId(333), extent_id: 61 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(334), op: ExtentRepair { repair_id: ReconciliationId(334), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(335), op: ExtentReopen { repair_id: ReconciliationId(335), extent_id: 61 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(336), op: ExtentFlush { repair_id: ReconciliationId(336), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(337), op: ExtentClose { repair_id: ReconciliationId(337), extent_id: 119 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(338), op: ExtentRepair { repair_id: ReconciliationId(338), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(339), op: ExtentReopen { repair_id: ReconciliationId(339), extent_id: 119 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(340), op: ExtentFlush { repair_id: ReconciliationId(340), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(341), op: ExtentClose { repair_id: ReconciliationId(341), extent_id: 30 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(342), op: ExtentRepair { repair_id: ReconciliationId(342), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(343), op: ExtentReopen { repair_id: ReconciliationId(343), extent_id: 30 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(344), op: ExtentFlush { repair_id: ReconciliationId(344), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(345), op: ExtentClose { repair_id: ReconciliationId(345), extent_id: 46 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(346), op: ExtentRepair { repair_id: ReconciliationId(346), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(347), op: ExtentReopen { repair_id: ReconciliationId(347), extent_id: 46 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(348), op: ExtentFlush { repair_id: ReconciliationId(348), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(349), op: ExtentClose { repair_id: ReconciliationId(349), extent_id: 127 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(350), op: ExtentRepair { repair_id: ReconciliationId(350), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(351), op: ExtentReopen { repair_id: ReconciliationId(351), extent_id: 127 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(352), op: ExtentFlush { repair_id: ReconciliationId(352), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(353), op: ExtentClose { repair_id: ReconciliationId(353), extent_id: 99 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(354), op: ExtentRepair { repair_id: ReconciliationId(354), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(355), op: ExtentReopen { repair_id: ReconciliationId(355), extent_id: 99 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(356), op: ExtentFlush { repair_id: ReconciliationId(356), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(357), op: ExtentClose { repair_id: ReconciliationId(357), extent_id: 12 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(358), op: ExtentRepair { repair_id: ReconciliationId(358), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(359), op: ExtentReopen { repair_id: ReconciliationId(359), extent_id: 12 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(360), op: ExtentFlush { repair_id: ReconciliationId(360), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(361), op: ExtentClose { repair_id: ReconciliationId(361), extent_id: 34 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(362), op: ExtentRepair { repair_id: ReconciliationId(362), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(363), op: ExtentReopen { repair_id: ReconciliationId(363), extent_id: 34 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(364), op: ExtentFlush { repair_id: ReconciliationId(364), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(365), op: ExtentClose { repair_id: ReconciliationId(365), extent_id: 83 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(366), op: ExtentRepair { repair_id: ReconciliationId(366), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(367), op: ExtentReopen { repair_id: ReconciliationId(367), extent_id: 83 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(368), op: ExtentFlush { repair_id: ReconciliationId(368), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(369), op: ExtentClose { repair_id: ReconciliationId(369), extent_id: 90 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(370), op: ExtentRepair { repair_id: ReconciliationId(370), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(371), op: ExtentReopen { repair_id: ReconciliationId(371), extent_id: 90 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(372), op: ExtentFlush { repair_id: ReconciliationId(372), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(373), op: ExtentClose { repair_id: ReconciliationId(373), extent_id: 66 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(374), op: ExtentRepair { repair_id: ReconciliationId(374), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(375), op: ExtentReopen { repair_id: ReconciliationId(375), extent_id: 66 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(376), op: ExtentFlush { repair_id: ReconciliationId(376), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(377), op: ExtentClose { repair_id: ReconciliationId(377), extent_id: 87 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(378), op: ExtentRepair { repair_id: ReconciliationId(378), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(379), op: ExtentReopen { repair_id: ReconciliationId(379), extent_id: 87 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(380), op: ExtentFlush { repair_id: ReconciliationId(380), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(381), op: ExtentClose { repair_id: ReconciliationId(381), extent_id: 139 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(382), op: ExtentRepair { repair_id: ReconciliationId(382), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(383), op: ExtentReopen { repair_id: ReconciliationId(383), extent_id: 139 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(384), op: ExtentFlush { repair_id: ReconciliationId(384), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(385), op: ExtentClose { repair_id: ReconciliationId(385), extent_id: 26 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(386), op: ExtentRepair { repair_id: ReconciliationId(386), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(387), op: ExtentReopen { repair_id: ReconciliationId(387), extent_id: 26 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(388), op: ExtentFlush { repair_id: ReconciliationId(388), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(389), op: ExtentClose { repair_id: ReconciliationId(389), extent_id: 84 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(390), op: ExtentRepair { repair_id: ReconciliationId(390), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(391), op: ExtentReopen { repair_id: ReconciliationId(391), extent_id: 84 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(392), op: ExtentFlush { repair_id: ReconciliationId(392), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(393), op: ExtentClose { repair_id: ReconciliationId(393), extent_id: 56 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(394), op: ExtentRepair { repair_id: ReconciliationId(394), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(395), op: ExtentReopen { repair_id: ReconciliationId(395), extent_id: 56 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(396), op: ExtentFlush { repair_id: ReconciliationId(396), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(397), op: ExtentClose { repair_id: ReconciliationId(397), extent_id: 94 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(398), op: ExtentRepair { repair_id: ReconciliationId(398), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(399), op: ExtentReopen { repair_id: ReconciliationId(399), extent_id: 94 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(400), op: ExtentFlush { repair_id: ReconciliationId(400), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(401), op: ExtentClose { repair_id: ReconciliationId(401), extent_id: 93 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(402), op: ExtentRepair { repair_id: ReconciliationId(402), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(403), op: ExtentReopen { repair_id: ReconciliationId(403), extent_id: 93 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(404), op: ExtentFlush { repair_id: ReconciliationId(404), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(405), op: ExtentClose { repair_id: ReconciliationId(405), extent_id: 44 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(406), op: ExtentRepair { repair_id: ReconciliationId(406), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(407), op: ExtentReopen { repair_id: ReconciliationId(407), extent_id: 44 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(408), op: ExtentFlush { repair_id: ReconciliationId(408), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(409), op: ExtentClose { repair_id: ReconciliationId(409), extent_id: 45 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(410), op: ExtentRepair { repair_id: ReconciliationId(410), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(411), op: ExtentReopen { repair_id: ReconciliationId(411), extent_id: 45 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(412), op: ExtentFlush { repair_id: ReconciliationId(412), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(413), op: ExtentClose { repair_id: ReconciliationId(413), extent_id: 115 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(414), op: ExtentRepair { repair_id: ReconciliationId(414), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(415), op: ExtentReopen { repair_id: ReconciliationId(415), extent_id: 115 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(416), op: ExtentFlush { repair_id: ReconciliationId(416), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(417), op: ExtentClose { repair_id: ReconciliationId(417), extent_id: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(418), op: ExtentRepair { repair_id: ReconciliationId(418), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(419), op: ExtentReopen { repair_id: ReconciliationId(419), extent_id: 32 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(420), op: ExtentFlush { repair_id: ReconciliationId(420), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(421), op: ExtentClose { repair_id: ReconciliationId(421), extent_id: 141 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(422), op: ExtentRepair { repair_id: ReconciliationId(422), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(423), op: ExtentReopen { repair_id: ReconciliationId(423), extent_id: 141 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(424), op: ExtentFlush { repair_id: ReconciliationId(424), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(425), op: ExtentClose { repair_id: ReconciliationId(425), extent_id: 9 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(426), op: ExtentRepair { repair_id: ReconciliationId(426), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(427), op: ExtentReopen { repair_id: ReconciliationId(427), extent_id: 9 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(428), op: ExtentFlush { repair_id: ReconciliationId(428), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(429), op: ExtentClose { repair_id: ReconciliationId(429), extent_id: 186 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(430), op: ExtentRepair { repair_id: ReconciliationId(430), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(431), op: ExtentReopen { repair_id: ReconciliationId(431), extent_id: 186 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(432), op: ExtentFlush { repair_id: ReconciliationId(432), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(433), op: ExtentClose { repair_id: ReconciliationId(433), extent_id: 130 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(434), op: ExtentRepair { repair_id: ReconciliationId(434), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(435), op: ExtentReopen { repair_id: ReconciliationId(435), extent_id: 130 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(436), op: ExtentFlush { repair_id: ReconciliationId(436), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(437), op: ExtentClose { repair_id: ReconciliationId(437), extent_id: 122 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(438), op: ExtentRepair { repair_id: ReconciliationId(438), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(439), op: ExtentReopen { repair_id: ReconciliationId(439), extent_id: 122 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(440), op: ExtentFlush { repair_id: ReconciliationId(440), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(441), op: ExtentClose { repair_id: ReconciliationId(441), extent_id: 105 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(442), op: ExtentRepair { repair_id: ReconciliationId(442), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(443), op: ExtentReopen { repair_id: ReconciliationId(443), extent_id: 105 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(444), op: ExtentFlush { repair_id: ReconciliationId(444), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(445), op: ExtentClose { repair_id: ReconciliationId(445), extent_id: 137 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(446), op: ExtentRepair { repair_id: ReconciliationId(446), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(447), op: ExtentReopen { repair_id: ReconciliationId(447), extent_id: 137 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(448), op: ExtentFlush { repair_id: ReconciliationId(448), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(449), op: ExtentClose { repair_id: ReconciliationId(449), extent_id: 7 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(450), op: ExtentRepair { repair_id: ReconciliationId(450), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(451), op: ExtentReopen { repair_id: ReconciliationId(451), extent_id: 7 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(452), op: ExtentFlush { repair_id: ReconciliationId(452), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(453), op: ExtentClose { repair_id: ReconciliationId(453), extent_id: 63 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(454), op: ExtentRepair { repair_id: ReconciliationId(454), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(455), op: ExtentReopen { repair_id: ReconciliationId(455), extent_id: 63 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(456), op: ExtentFlush { repair_id: ReconciliationId(456), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(457), op: ExtentClose { repair_id: ReconciliationId(457), extent_id: 58 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(458), op: ExtentRepair { repair_id: ReconciliationId(458), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(459), op: ExtentReopen { repair_id: ReconciliationId(459), extent_id: 58 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(460), op: ExtentFlush { repair_id: ReconciliationId(460), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(461), op: ExtentClose { repair_id: ReconciliationId(461), extent_id: 96 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(462), op: ExtentRepair { repair_id: ReconciliationId(462), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(463), op: ExtentReopen { repair_id: ReconciliationId(463), extent_id: 96 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(464), op: ExtentFlush { repair_id: ReconciliationId(464), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(465), op: ExtentClose { repair_id: ReconciliationId(465), extent_id: 40 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(466), op: ExtentRepair { repair_id: ReconciliationId(466), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(467), op: ExtentReopen { repair_id: ReconciliationId(467), extent_id: 40 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(468), op: ExtentFlush { repair_id: ReconciliationId(468), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(469), op: ExtentClose { repair_id: ReconciliationId(469), extent_id: 80 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(470), op: ExtentRepair { repair_id: ReconciliationId(470), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(471), op: ExtentReopen { repair_id: ReconciliationId(471), extent_id: 80 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(472), op: ExtentFlush { repair_id: ReconciliationId(472), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(473), op: ExtentClose { repair_id: ReconciliationId(473), extent_id: 149 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(474), op: ExtentRepair { repair_id: ReconciliationId(474), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(475), op: ExtentReopen { repair_id: ReconciliationId(475), extent_id: 149 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(476), op: ExtentFlush { repair_id: ReconciliationId(476), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(477), op: ExtentClose { repair_id: ReconciliationId(477), extent_id: 161 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(478), op: ExtentRepair { repair_id: ReconciliationId(478), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(479), op: ExtentReopen { repair_id: ReconciliationId(479), extent_id: 161 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(480), op: ExtentFlush { repair_id: ReconciliationId(480), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(481), op: ExtentClose { repair_id: ReconciliationId(481), extent_id: 71 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(482), op: ExtentRepair { repair_id: ReconciliationId(482), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(483), op: ExtentReopen { repair_id: ReconciliationId(483), extent_id: 71 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(484), op: ExtentFlush { repair_id: ReconciliationId(484), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(485), op: ExtentClose { repair_id: ReconciliationId(485), extent_id: 5 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(486), op: ExtentRepair { repair_id: ReconciliationId(486), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(487), op: ExtentReopen { repair_id: ReconciliationId(487), extent_id: 5 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(488), op: ExtentFlush { repair_id: ReconciliationId(488), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(489), op: ExtentClose { repair_id: ReconciliationId(489), extent_id: 23 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(490), op: ExtentRepair { repair_id: ReconciliationId(490), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(491), op: ExtentReopen { repair_id: ReconciliationId(491), extent_id: 23 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(492), op: ExtentFlush { repair_id: ReconciliationId(492), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(493), op: ExtentClose { repair_id: ReconciliationId(493), extent_id: 98 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(494), op: ExtentRepair { repair_id: ReconciliationId(494), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(495), op: ExtentReopen { repair_id: ReconciliationId(495), extent_id: 98 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(496), op: ExtentFlush { repair_id: ReconciliationId(496), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(497), op: ExtentClose { repair_id: ReconciliationId(497), extent_id: 1 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(498), op: ExtentRepair { repair_id: ReconciliationId(498), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(499), op: ExtentReopen { repair_id: ReconciliationId(499), extent_id: 1 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(500), op: ExtentFlush { repair_id: ReconciliationId(500), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(501), op: ExtentClose { repair_id: ReconciliationId(501), extent_id: 166 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(502), op: ExtentRepair { repair_id: ReconciliationId(502), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(503), op: ExtentReopen { repair_id: ReconciliationId(503), extent_id: 166 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(504), op: ExtentFlush { repair_id: ReconciliationId(504), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(505), op: ExtentClose { repair_id: ReconciliationId(505), extent_id: 185 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(506), op: ExtentRepair { repair_id: ReconciliationId(506), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(507), op: ExtentReopen { repair_id: ReconciliationId(507), extent_id: 185 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(508), op: ExtentFlush { repair_id: ReconciliationId(508), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(509), op: ExtentClose { repair_id: ReconciliationId(509), extent_id: 86 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(510), op: ExtentRepair { repair_id: ReconciliationId(510), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(511), op: ExtentReopen { repair_id: ReconciliationId(511), extent_id: 86 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(512), op: ExtentFlush { repair_id: ReconciliationId(512), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(513), op: ExtentClose { repair_id: ReconciliationId(513), extent_id: 135 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(514), op: ExtentRepair { repair_id: ReconciliationId(514), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(515), op: ExtentReopen { repair_id: ReconciliationId(515), extent_id: 135 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(516), op: ExtentFlush { repair_id: ReconciliationId(516), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(517), op: ExtentClose { repair_id: ReconciliationId(517), extent_id: 13 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(518), op: ExtentRepair { repair_id: ReconciliationId(518), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(519), op: ExtentReopen { repair_id: ReconciliationId(519), extent_id: 13 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(520), op: ExtentFlush { repair_id: ReconciliationId(520), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(521), op: ExtentClose { repair_id: ReconciliationId(521), extent_id: 29 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(522), op: ExtentRepair { repair_id: ReconciliationId(522), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(523), op: ExtentReopen { repair_id: ReconciliationId(523), extent_id: 29 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(524), op: ExtentFlush { repair_id: ReconciliationId(524), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(525), op: ExtentClose { repair_id: ReconciliationId(525), extent_id: 27 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(526), op: ExtentRepair { repair_id: ReconciliationId(526), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(527), op: ExtentReopen { repair_id: ReconciliationId(527), extent_id: 27 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(528), op: ExtentFlush { repair_id: ReconciliationId(528), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(529), op: ExtentClose { repair_id: ReconciliationId(529), extent_id: 187 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(530), op: ExtentRepair { repair_id: ReconciliationId(530), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(531), op: ExtentReopen { repair_id: ReconciliationId(531), extent_id: 187 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(532), op: ExtentFlush { repair_id: ReconciliationId(532), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(533), op: ExtentClose { repair_id: ReconciliationId(533), extent_id: 10 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(534), op: ExtentRepair { repair_id: ReconciliationId(534), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(535), op: ExtentReopen { repair_id: ReconciliationId(535), extent_id: 10 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(536), op: ExtentFlush { repair_id: ReconciliationId(536), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(537), op: ExtentClose { repair_id: ReconciliationId(537), extent_id: 102 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(538), op: ExtentRepair { repair_id: ReconciliationId(538), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(539), op: ExtentReopen { repair_id: ReconciliationId(539), extent_id: 102 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(540), op: ExtentFlush { repair_id: ReconciliationId(540), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(541), op: ExtentClose { repair_id: ReconciliationId(541), extent_id: 85 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(542), op: ExtentRepair { repair_id: ReconciliationId(542), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(543), op: ExtentReopen { repair_id: ReconciliationId(543), extent_id: 85 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(544), op: ExtentFlush { repair_id: ReconciliationId(544), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(545), op: ExtentClose { repair_id: ReconciliationId(545), extent_id: 100 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(546), op: ExtentRepair { repair_id: ReconciliationId(546), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(547), op: ExtentReopen { repair_id: ReconciliationId(547), extent_id: 100 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(548), op: ExtentFlush { repair_id: ReconciliationId(548), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(549), op: ExtentClose { repair_id: ReconciliationId(549), extent_id: 95 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(550), op: ExtentRepair { repair_id: ReconciliationId(550), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(551), op: ExtentReopen { repair_id: ReconciliationId(551), extent_id: 95 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(552), op: ExtentFlush { repair_id: ReconciliationId(552), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(553), op: ExtentClose { repair_id: ReconciliationId(553), extent_id: 154 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(554), op: ExtentRepair { repair_id: ReconciliationId(554), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(555), op: ExtentReopen { repair_id: ReconciliationId(555), extent_id: 154 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(556), op: ExtentFlush { repair_id: ReconciliationId(556), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(557), op: ExtentClose { repair_id: ReconciliationId(557), extent_id: 50 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(558), op: ExtentRepair { repair_id: ReconciliationId(558), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(559), op: ExtentReopen { repair_id: ReconciliationId(559), extent_id: 50 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(560), op: ExtentFlush { repair_id: ReconciliationId(560), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(561), op: ExtentClose { repair_id: ReconciliationId(561), extent_id: 91 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(562), op: ExtentRepair { repair_id: ReconciliationId(562), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(563), op: ExtentReopen { repair_id: ReconciliationId(563), extent_id: 91 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(564), op: ExtentFlush { repair_id: ReconciliationId(564), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(565), op: ExtentClose { repair_id: ReconciliationId(565), extent_id: 158 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(566), op: ExtentRepair { repair_id: ReconciliationId(566), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(567), op: ExtentReopen { repair_id: ReconciliationId(567), extent_id: 158 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(568), op: ExtentFlush { repair_id: ReconciliationId(568), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(569), op: ExtentClose { repair_id: ReconciliationId(569), extent_id: 177 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(570), op: ExtentRepair { repair_id: ReconciliationId(570), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(571), op: ExtentReopen { repair_id: ReconciliationId(571), extent_id: 177 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(572), op: ExtentFlush { repair_id: ReconciliationId(572), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(573), op: ExtentClose { repair_id: ReconciliationId(573), extent_id: 134 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(574), op: ExtentRepair { repair_id: ReconciliationId(574), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(575), op: ExtentReopen { repair_id: ReconciliationId(575), extent_id: 134 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(576), op: ExtentFlush { repair_id: ReconciliationId(576), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(577), op: ExtentClose { repair_id: ReconciliationId(577), extent_id: 69 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(578), op: ExtentRepair { repair_id: ReconciliationId(578), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(579), op: ExtentReopen { repair_id: ReconciliationId(579), extent_id: 69 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(580), op: ExtentFlush { repair_id: ReconciliationId(580), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(581), op: ExtentClose { repair_id: ReconciliationId(581), extent_id: 155 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(582), op: ExtentRepair { repair_id: ReconciliationId(582), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(583), op: ExtentReopen { repair_id: ReconciliationId(583), extent_id: 155 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(584), op: ExtentFlush { repair_id: ReconciliationId(584), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(585), op: ExtentClose { repair_id: ReconciliationId(585), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(586), op: ExtentRepair { repair_id: ReconciliationId(586), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(587), op: ExtentReopen { repair_id: ReconciliationId(587), extent_id: 0 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(588), op: ExtentFlush { repair_id: ReconciliationId(588), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(589), op: ExtentClose { repair_id: ReconciliationId(589), extent_id: 164 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(590), op: ExtentRepair { repair_id: ReconciliationId(590), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(591), op: ExtentReopen { repair_id: ReconciliationId(591), extent_id: 164 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(592), op: ExtentFlush { repair_id: ReconciliationId(592), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(593), op: ExtentClose { repair_id: ReconciliationId(593), extent_id: 24 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(594), op: ExtentRepair { repair_id: ReconciliationId(594), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(595), op: ExtentReopen { repair_id: ReconciliationId(595), extent_id: 24 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(596), op: ExtentFlush { repair_id: ReconciliationId(596), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(597), op: ExtentClose { repair_id: ReconciliationId(597), extent_id: 39 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(598), op: ExtentRepair { repair_id: ReconciliationId(598), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(599), op: ExtentReopen { repair_id: ReconciliationId(599), extent_id: 39 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(600), op: ExtentFlush { repair_id: ReconciliationId(600), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(601), op: ExtentClose { repair_id: ReconciliationId(601), extent_id: 48 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(602), op: ExtentRepair { repair_id: ReconciliationId(602), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(603), op: ExtentReopen { repair_id: ReconciliationId(603), extent_id: 48 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(604), op: ExtentFlush { repair_id: ReconciliationId(604), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(605), op: ExtentClose { repair_id: ReconciliationId(605), extent_id: 16 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(606), op: ExtentRepair { repair_id: ReconciliationId(606), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(607), op: ExtentReopen { repair_id: ReconciliationId(607), extent_id: 16 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(608), op: ExtentFlush { repair_id: ReconciliationId(608), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(609), op: ExtentClose { repair_id: ReconciliationId(609), extent_id: 4 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(610), op: ExtentRepair { repair_id: ReconciliationId(610), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(611), op: ExtentReopen { repair_id: ReconciliationId(611), extent_id: 4 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(612), op: ExtentFlush { repair_id: ReconciliationId(612), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(613), op: ExtentClose { repair_id: ReconciliationId(613), extent_id: 70 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(614), op: ExtentRepair { repair_id: ReconciliationId(614), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(615), op: ExtentReopen { repair_id: ReconciliationId(615), extent_id: 70 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(616), op: ExtentFlush { repair_id: ReconciliationId(616), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(617), op: ExtentClose { repair_id: ReconciliationId(617), extent_id: 76 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(618), op: ExtentRepair { repair_id: ReconciliationId(618), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(619), op: ExtentReopen { repair_id: ReconciliationId(619), extent_id: 76 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(620), op: ExtentFlush { repair_id: ReconciliationId(620), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(621), op: ExtentClose { repair_id: ReconciliationId(621), extent_id: 113 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(622), op: ExtentRepair { repair_id: ReconciliationId(622), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(623), op: ExtentReopen { repair_id: ReconciliationId(623), extent_id: 113 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(624), op: ExtentFlush { repair_id: ReconciliationId(624), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(625), op: ExtentClose { repair_id: ReconciliationId(625), extent_id: 108 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(626), op: ExtentRepair { repair_id: ReconciliationId(626), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(627), op: ExtentReopen { repair_id: ReconciliationId(627), extent_id: 108 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(628), op: ExtentFlush { repair_id: ReconciliationId(628), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(629), op: ExtentClose { repair_id: ReconciliationId(629), extent_id: 42 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(630), op: ExtentRepair { repair_id: ReconciliationId(630), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(631), op: ExtentReopen { repair_id: ReconciliationId(631), extent_id: 42 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(632), op: ExtentFlush { repair_id: ReconciliationId(632), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(633), op: ExtentClose { repair_id: ReconciliationId(633), extent_id: 140 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(634), op: ExtentRepair { repair_id: ReconciliationId(634), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(635), op: ExtentReopen { repair_id: ReconciliationId(635), extent_id: 140 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(636), op: ExtentFlush { repair_id: ReconciliationId(636), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(637), op: ExtentClose { repair_id: ReconciliationId(637), extent_id: 178 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(638), op: ExtentRepair { repair_id: ReconciliationId(638), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(639), op: ExtentReopen { repair_id: ReconciliationId(639), extent_id: 178 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(640), op: ExtentFlush { repair_id: ReconciliationId(640), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(641), op: ExtentClose { repair_id: ReconciliationId(641), extent_id: 181 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(642), op: ExtentRepair { repair_id: ReconciliationId(642), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(643), op: ExtentReopen { repair_id: ReconciliationId(643), extent_id: 181 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(644), op: ExtentFlush { repair_id: ReconciliationId(644), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(645), op: ExtentClose { repair_id: ReconciliationId(645), extent_id: 82 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(646), op: ExtentRepair { repair_id: ReconciliationId(646), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(647), op: ExtentReopen { repair_id: ReconciliationId(647), extent_id: 82 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(648), op: ExtentFlush { repair_id: ReconciliationId(648), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(649), op: ExtentClose { repair_id: ReconciliationId(649), extent_id: 174 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(650), op: ExtentRepair { repair_id: ReconciliationId(650), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(651), op: ExtentReopen { repair_id: ReconciliationId(651), extent_id: 174 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(652), op: ExtentFlush { repair_id: ReconciliationId(652), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(653), op: ExtentClose { repair_id: ReconciliationId(653), extent_id: 17 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(654), op: ExtentRepair { repair_id: ReconciliationId(654), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(655), op: ExtentReopen { repair_id: ReconciliationId(655), extent_id: 17 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(656), op: ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(657), op: ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(658), op: ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(659), op: ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(660), op: ExtentFlush { repair_id: ReconciliationId(660), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(661), op: ExtentClose { repair_id: ReconciliationId(661), extent_id: 142 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(662), op: ExtentRepair { repair_id: ReconciliationId(662), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(663), op: ExtentReopen { repair_id: ReconciliationId(663), extent_id: 142 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(664), op: ExtentFlush { repair_id: ReconciliationId(664), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(665), op: ExtentClose { repair_id: ReconciliationId(665), extent_id: 88 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(666), op: ExtentRepair { repair_id: ReconciliationId(666), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(667), op: ExtentReopen { repair_id: ReconciliationId(667), extent_id: 88 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(668), op: ExtentFlush { repair_id: ReconciliationId(668), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(669), op: ExtentClose { repair_id: ReconciliationId(669), extent_id: 51 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(670), op: ExtentRepair { repair_id: ReconciliationId(670), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(671), op: ExtentReopen { repair_id: ReconciliationId(671), extent_id: 51 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(672), op: ExtentFlush { repair_id: ReconciliationId(672), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(673), op: ExtentClose { repair_id: ReconciliationId(673), extent_id: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(674), op: ExtentRepair { repair_id: ReconciliationId(674), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(675), op: ExtentReopen { repair_id: ReconciliationId(675), extent_id: 33 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(676), op: ExtentFlush { repair_id: ReconciliationId(676), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(677), op: ExtentClose { repair_id: ReconciliationId(677), extent_id: 54 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(678), op: ExtentRepair { repair_id: ReconciliationId(678), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(679), op: ExtentReopen { repair_id: ReconciliationId(679), extent_id: 54 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(680), op: ExtentFlush { repair_id: ReconciliationId(680), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(681), op: ExtentClose { repair_id: ReconciliationId(681), extent_id: 112 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(682), op: ExtentRepair { repair_id: ReconciliationId(682), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(683), op: ExtentReopen { repair_id: ReconciliationId(683), extent_id: 112 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(684), op: ExtentFlush { repair_id: ReconciliationId(684), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(685), op: ExtentClose { repair_id: ReconciliationId(685), extent_id: 143 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(686), op: ExtentRepair { repair_id: ReconciliationId(686), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(687), op: ExtentReopen { repair_id: ReconciliationId(687), extent_id: 143 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(688), op: ExtentFlush { repair_id: ReconciliationId(688), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(689), op: ExtentClose { repair_id: ReconciliationId(689), extent_id: 151 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(690), op: ExtentRepair { repair_id: ReconciliationId(690), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(691), op: ExtentReopen { repair_id: ReconciliationId(691), extent_id: 151 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(692), op: ExtentFlush { repair_id: ReconciliationId(692), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(693), op: ExtentClose { repair_id: ReconciliationId(693), extent_id: 101 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(694), op: ExtentRepair { repair_id: ReconciliationId(694), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(695), op: ExtentReopen { repair_id: ReconciliationId(695), extent_id: 101 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(696), op: ExtentFlush { repair_id: ReconciliationId(696), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(697), op: ExtentClose { repair_id: ReconciliationId(697), extent_id: 173 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(698), op: ExtentRepair { repair_id: ReconciliationId(698), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(699), op: ExtentReopen { repair_id: ReconciliationId(699), extent_id: 173 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(700), op: ExtentFlush { repair_id: ReconciliationId(700), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(701), op: ExtentClose { repair_id: ReconciliationId(701), extent_id: 20 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(702), op: ExtentRepair { repair_id: ReconciliationId(702), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(703), op: ExtentReopen { repair_id: ReconciliationId(703), extent_id: 20 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(704), op: ExtentFlush { repair_id: ReconciliationId(704), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(705), op: ExtentClose { repair_id: ReconciliationId(705), extent_id: 104 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(706), op: ExtentRepair { repair_id: ReconciliationId(706), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(707), op: ExtentReopen { repair_id: ReconciliationId(707), extent_id: 104 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(708), op: ExtentFlush { repair_id: ReconciliationId(708), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(709), op: ExtentClose { repair_id: ReconciliationId(709), extent_id: 81 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(710), op: ExtentRepair { repair_id: ReconciliationId(710), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(711), op: ExtentReopen { repair_id: ReconciliationId(711), extent_id: 81 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(712), op: ExtentFlush { repair_id: ReconciliationId(712), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(713), op: ExtentClose { repair_id: ReconciliationId(713), extent_id: 120 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(714), op: ExtentRepair { repair_id: ReconciliationId(714), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(715), op: ExtentReopen { repair_id: ReconciliationId(715), extent_id: 120 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(716), op: ExtentFlush { repair_id: ReconciliationId(716), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(717), op: ExtentClose { repair_id: ReconciliationId(717), extent_id: 145 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(718), op: ExtentRepair { repair_id: ReconciliationId(718), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(719), op: ExtentReopen { repair_id: ReconciliationId(719), extent_id: 145 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(720), op: ExtentFlush { repair_id: ReconciliationId(720), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(721), op: ExtentClose { repair_id: ReconciliationId(721), extent_id: 131 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(722), op: ExtentRepair { repair_id: ReconciliationId(722), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(723), op: ExtentReopen { repair_id: ReconciliationId(723), extent_id: 131 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(724), op: ExtentFlush { repair_id: ReconciliationId(724), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(725), op: ExtentClose { repair_id: ReconciliationId(725), extent_id: 170 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(726), op: ExtentRepair { repair_id: ReconciliationId(726), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(727), op: ExtentReopen { repair_id: ReconciliationId(727), extent_id: 170 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(728), op: ExtentFlush { repair_id: ReconciliationId(728), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(729), op: ExtentClose { repair_id: ReconciliationId(729), extent_id: 72 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(730), op: ExtentRepair { repair_id: ReconciliationId(730), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(731), op: ExtentReopen { repair_id: ReconciliationId(731), extent_id: 72 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(732), op: ExtentFlush { repair_id: ReconciliationId(732), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(733), op: ExtentClose { repair_id: ReconciliationId(733), extent_id: 180 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(734), op: ExtentRepair { repair_id: ReconciliationId(734), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(735), op: ExtentReopen { repair_id: ReconciliationId(735), extent_id: 180 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(736), op: ExtentFlush { repair_id: ReconciliationId(736), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(737), op: ExtentClose { repair_id: ReconciliationId(737), extent_id: 57 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(738), op: ExtentRepair { repair_id: ReconciliationId(738), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(739), op: ExtentReopen { repair_id: ReconciliationId(739), extent_id: 57 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(740), op: ExtentFlush { repair_id: ReconciliationId(740), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(741), op: ExtentClose { repair_id: ReconciliationId(741), extent_id: 14 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(742), op: ExtentRepair { repair_id: ReconciliationId(742), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(743), op: ExtentReopen { repair_id: ReconciliationId(743), extent_id: 14 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(744), op: ExtentFlush { repair_id: ReconciliationId(744), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(745), op: ExtentClose { repair_id: ReconciliationId(745), extent_id: 8 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(746), op: ExtentRepair { repair_id: ReconciliationId(746), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(747), op: ExtentReopen { repair_id: ReconciliationId(747), extent_id: 8 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(748), op: ExtentFlush { repair_id: ReconciliationId(748), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(749), op: ExtentClose { repair_id: ReconciliationId(749), extent_id: 114 }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(750), op: ExtentRepair { repair_id: ReconciliationId(750), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }, ReconcileIO { id: ReconciliationId(751), op: ExtentReopen { repair_id: ReconciliationId(751), extent_id: 114 }, state: ClientData([New, New, New]) }], : downstairs
Sep 22 23:22:14.135 INFO Begin repair with 752 commands
Sep 22 23:22:14.135 INFO Pop front: ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.135 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.135 INFO [0] received reconcile message
Sep 22 23:22:14.135 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.135 INFO [0] client ExtentFlush { repair_id: ReconciliationId(0), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.135 INFO [1] received reconcile message
Sep 22 23:22:14.135 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.135 INFO [1] client ExtentFlush { repair_id: ReconciliationId(0), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.135 INFO [2] received reconcile message
Sep 22 23:22:14.135 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(0), op: ExtentFlush { repair_id: ReconciliationId(0), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.135 INFO [2] client ExtentFlush { repair_id: ReconciliationId(0), extent_id: 31, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.135 DEBG 0 Flush extent 31 with f:2 g:2
Sep 22 23:22:14.135 DEBG Flush just extent 31 with f:2 and g:2
Sep 22 23:22:14.135 DEBG [1] It's time to notify for 0
Sep 22 23:22:14.135 INFO Completion from [1] id:0 status:true
Sep 22 23:22:14.135 INFO [1/752] Repair commands completed
Sep 22 23:22:14.135 INFO Pop front: ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 31 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.135 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.136 INFO [0] received reconcile message
Sep 22 23:22:14.136 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 31 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.136 INFO [0] client ExtentClose { repair_id: ReconciliationId(1), extent_id: 31 }
Sep 22 23:22:14.136 INFO [1] received reconcile message
Sep 22 23:22:14.136 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 31 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.136 INFO [1] client ExtentClose { repair_id: ReconciliationId(1), extent_id: 31 }
Sep 22 23:22:14.136 INFO [2] received reconcile message
Sep 22 23:22:14.136 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(1), op: ExtentClose { repair_id: ReconciliationId(1), extent_id: 31 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.136 INFO [2] client ExtentClose { repair_id: ReconciliationId(1), extent_id: 31 }
Sep 22 23:22:14.136 DEBG 1 Close extent 31
Sep 22 23:22:14.136 DEBG 1 Close extent 31
Sep 22 23:22:14.136 DEBG 1 Close extent 31
Sep 22 23:22:14.137 DEBG [2] It's time to notify for 1
Sep 22 23:22:14.137 INFO Completion from [2] id:1 status:true
Sep 22 23:22:14.137 INFO [2/752] Repair commands completed
Sep 22 23:22:14.137 INFO Pop front: ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.137 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.137 INFO [0] received reconcile message
Sep 22 23:22:14.137 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.137 INFO [0] client ExtentRepair { repair_id: ReconciliationId(2), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.137 INFO [0] Sending repair request ReconciliationId(2)
Sep 22 23:22:14.137 INFO [1] received reconcile message
Sep 22 23:22:14.137 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.137 INFO [1] client ExtentRepair { repair_id: ReconciliationId(2), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.137 INFO [1] No action required ReconciliationId(2)
Sep 22 23:22:14.137 INFO [2] received reconcile message
Sep 22 23:22:14.137 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(2), op: ExtentRepair { repair_id: ReconciliationId(2), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.137 INFO [2] client ExtentRepair { repair_id: ReconciliationId(2), extent_id: 31, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.137 INFO [2] No action required ReconciliationId(2)
Sep 22 23:22:14.137 DEBG 2 Repair extent 31 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.137 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/01F.copy"
Sep 22 23:22:14.202 INFO accepted connection, remote_addr: 127.0.0.1:60599, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.203 TRCE incoming request, uri: /extent/31/files, method: GET, req_id: dbda508a-9d8a-485a-932c-0c4e0c18b908, remote_addr: 127.0.0.1:60599, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.203 INFO request completed, latency_us: 270, response_code: 200, uri: /extent/31/files, method: GET, req_id: dbda508a-9d8a-485a-932c-0c4e0c18b908, remote_addr: 127.0.0.1:60599, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.203 INFO eid:31 Found repair files: ["01F", "01F.db"]
Sep 22 23:22:14.204 TRCE incoming request, uri: /newextent/31/data, method: GET, req_id: 077849bd-4094-4a21-a79a-6f3d77680cf1, remote_addr: 127.0.0.1:60599, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.204 INFO request completed, latency_us: 477, response_code: 200, uri: /newextent/31/data, method: GET, req_id: 077849bd-4094-4a21-a79a-6f3d77680cf1, remote_addr: 127.0.0.1:60599, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.209 TRCE incoming request, uri: /newextent/31/db, method: GET, req_id: 0bba777e-9711-4188-88f2-2c5eb30c4f88, remote_addr: 127.0.0.1:60599, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.209 INFO request completed, latency_us: 289, response_code: 200, uri: /newextent/31/db, method: GET, req_id: 0bba777e-9711-4188-88f2-2c5eb30c4f88, remote_addr: 127.0.0.1:60599, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.211 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/01F.copy" to "/tmp/downstairs-zrMnlo6G/00/000/01F.replace"
Sep 22 23:22:14.211 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.212 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/01F.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.212 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01F"
Sep 22 23:22:14.212 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01F.db"
Sep 22 23:22:14.212 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.212 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/01F.replace" to "/tmp/downstairs-zrMnlo6G/00/000/01F.completed"
Sep 22 23:22:14.212 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.212 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.213 DEBG [0] It's time to notify for 2
Sep 22 23:22:14.213 INFO Completion from [0] id:2 status:true
Sep 22 23:22:14.213 INFO [3/752] Repair commands completed
Sep 22 23:22:14.213 INFO Pop front: ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 31 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.213 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.213 INFO [0] received reconcile message
Sep 22 23:22:14.213 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 31 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.213 INFO [0] client ExtentReopen { repair_id: ReconciliationId(3), extent_id: 31 }
Sep 22 23:22:14.213 INFO [1] received reconcile message
Sep 22 23:22:14.213 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 31 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.213 INFO [1] client ExtentReopen { repair_id: ReconciliationId(3), extent_id: 31 }
Sep 22 23:22:14.213 INFO [2] received reconcile message
Sep 22 23:22:14.213 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(3), op: ExtentReopen { repair_id: ReconciliationId(3), extent_id: 31 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.213 INFO [2] client ExtentReopen { repair_id: ReconciliationId(3), extent_id: 31 }
Sep 22 23:22:14.213 DEBG 3 Reopen extent 31
Sep 22 23:22:14.214 DEBG 3 Reopen extent 31
Sep 22 23:22:14.214 DEBG 3 Reopen extent 31
Sep 22 23:22:14.215 DEBG [2] It's time to notify for 3
Sep 22 23:22:14.215 INFO Completion from [2] id:3 status:true
Sep 22 23:22:14.215 INFO [4/752] Repair commands completed
Sep 22 23:22:14.215 INFO Pop front: ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.215 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.215 INFO [0] received reconcile message
Sep 22 23:22:14.215 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.215 INFO [0] client ExtentFlush { repair_id: ReconciliationId(4), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.215 INFO [1] received reconcile message
Sep 22 23:22:14.215 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.215 INFO [1] client ExtentFlush { repair_id: ReconciliationId(4), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.215 INFO [2] received reconcile message
Sep 22 23:22:14.215 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(4), op: ExtentFlush { repair_id: ReconciliationId(4), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.215 INFO [2] client ExtentFlush { repair_id: ReconciliationId(4), extent_id: 175, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.215 DEBG 4 Flush extent 175 with f:2 g:2
Sep 22 23:22:14.215 DEBG Flush just extent 175 with f:2 and g:2
Sep 22 23:22:14.216 DEBG [1] It's time to notify for 4
Sep 22 23:22:14.216 INFO Completion from [1] id:4 status:true
Sep 22 23:22:14.216 INFO [5/752] Repair commands completed
Sep 22 23:22:14.216 INFO Pop front: ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 175 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.216 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.216 INFO [0] received reconcile message
Sep 22 23:22:14.216 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 175 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.216 INFO [0] client ExtentClose { repair_id: ReconciliationId(5), extent_id: 175 }
Sep 22 23:22:14.216 INFO [1] received reconcile message
Sep 22 23:22:14.216 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 175 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.216 INFO [1] client ExtentClose { repair_id: ReconciliationId(5), extent_id: 175 }
Sep 22 23:22:14.216 INFO [2] received reconcile message
Sep 22 23:22:14.216 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(5), op: ExtentClose { repair_id: ReconciliationId(5), extent_id: 175 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.216 INFO [2] client ExtentClose { repair_id: ReconciliationId(5), extent_id: 175 }
Sep 22 23:22:14.216 DEBG 5 Close extent 175
Sep 22 23:22:14.216 DEBG 5 Close extent 175
Sep 22 23:22:14.217 DEBG 5 Close extent 175
Sep 22 23:22:14.217 DEBG [2] It's time to notify for 5
Sep 22 23:22:14.217 INFO Completion from [2] id:5 status:true
Sep 22 23:22:14.217 INFO [6/752] Repair commands completed
Sep 22 23:22:14.217 INFO Pop front: ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.217 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.217 INFO [0] received reconcile message
Sep 22 23:22:14.217 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.217 INFO [0] client ExtentRepair { repair_id: ReconciliationId(6), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.217 INFO [0] Sending repair request ReconciliationId(6)
Sep 22 23:22:14.217 INFO [1] received reconcile message
Sep 22 23:22:14.217 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.217 INFO [1] client ExtentRepair { repair_id: ReconciliationId(6), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.217 INFO [1] No action required ReconciliationId(6)
Sep 22 23:22:14.217 INFO [2] received reconcile message
Sep 22 23:22:14.217 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(6), op: ExtentRepair { repair_id: ReconciliationId(6), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.217 INFO [2] client ExtentRepair { repair_id: ReconciliationId(6), extent_id: 175, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.217 INFO [2] No action required ReconciliationId(6)
Sep 22 23:22:14.217 DEBG 6 Repair extent 175 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.217 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0AF.copy"
Sep 22 23:22:14.282 INFO accepted connection, remote_addr: 127.0.0.1:42863, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.282 TRCE incoming request, uri: /extent/175/files, method: GET, req_id: 6c12a7e7-5aa4-4668-8b41-ffe364262304, remote_addr: 127.0.0.1:42863, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.283 INFO request completed, latency_us: 208, response_code: 200, uri: /extent/175/files, method: GET, req_id: 6c12a7e7-5aa4-4668-8b41-ffe364262304, remote_addr: 127.0.0.1:42863, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.283 INFO eid:175 Found repair files: ["0AF", "0AF.db"]
Sep 22 23:22:14.283 TRCE incoming request, uri: /newextent/175/data, method: GET, req_id: 2686f34a-43e2-4d00-9de0-1e633fa0c75e, remote_addr: 127.0.0.1:42863, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.283 INFO request completed, latency_us: 324, response_code: 200, uri: /newextent/175/data, method: GET, req_id: 2686f34a-43e2-4d00-9de0-1e633fa0c75e, remote_addr: 127.0.0.1:42863, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.288 TRCE incoming request, uri: /newextent/175/db, method: GET, req_id: aa7671e6-dc37-4383-940b-858878f6e46d, remote_addr: 127.0.0.1:42863, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.289 INFO request completed, latency_us: 288, response_code: 200, uri: /newextent/175/db, method: GET, req_id: aa7671e6-dc37-4383-940b-858878f6e46d, remote_addr: 127.0.0.1:42863, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.290 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0AF.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0AF.replace"
Sep 22 23:22:14.290 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.291 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0AF.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.291 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AF"
Sep 22 23:22:14.291 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AF.db"
Sep 22 23:22:14.291 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.291 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0AF.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0AF.completed"
Sep 22 23:22:14.291 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.291 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.292 DEBG [0] It's time to notify for 6
Sep 22 23:22:14.292 INFO Completion from [0] id:6 status:true
Sep 22 23:22:14.292 INFO [7/752] Repair commands completed
Sep 22 23:22:14.292 INFO Pop front: ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 175 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.292 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.292 INFO [0] received reconcile message
Sep 22 23:22:14.292 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 175 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.292 INFO [0] client ExtentReopen { repair_id: ReconciliationId(7), extent_id: 175 }
Sep 22 23:22:14.292 INFO [1] received reconcile message
Sep 22 23:22:14.292 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 175 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.292 INFO [1] client ExtentReopen { repair_id: ReconciliationId(7), extent_id: 175 }
Sep 22 23:22:14.292 INFO [2] received reconcile message
Sep 22 23:22:14.292 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(7), op: ExtentReopen { repair_id: ReconciliationId(7), extent_id: 175 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.292 INFO [2] client ExtentReopen { repair_id: ReconciliationId(7), extent_id: 175 }
Sep 22 23:22:14.292 DEBG 7 Reopen extent 175
Sep 22 23:22:14.293 DEBG 7 Reopen extent 175
Sep 22 23:22:14.293 DEBG 7 Reopen extent 175
Sep 22 23:22:14.294 DEBG [2] It's time to notify for 7
Sep 22 23:22:14.294 INFO Completion from [2] id:7 status:true
Sep 22 23:22:14.294 INFO [8/752] Repair commands completed
Sep 22 23:22:14.294 INFO Pop front: ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.294 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.294 INFO [0] received reconcile message
Sep 22 23:22:14.294 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.294 INFO [0] client ExtentFlush { repair_id: ReconciliationId(8), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.294 INFO [1] received reconcile message
Sep 22 23:22:14.294 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.294 INFO [1] client ExtentFlush { repair_id: ReconciliationId(8), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.294 INFO [2] received reconcile message
Sep 22 23:22:14.294 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(8), op: ExtentFlush { repair_id: ReconciliationId(8), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.294 INFO [2] client ExtentFlush { repair_id: ReconciliationId(8), extent_id: 3, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.294 DEBG 8 Flush extent 3 with f:2 g:2
Sep 22 23:22:14.294 DEBG Flush just extent 3 with f:2 and g:2
Sep 22 23:22:14.294 DEBG [1] It's time to notify for 8
Sep 22 23:22:14.294 INFO Completion from [1] id:8 status:true
Sep 22 23:22:14.294 INFO [9/752] Repair commands completed
Sep 22 23:22:14.294 INFO Pop front: ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 3 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.294 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.294 INFO [0] received reconcile message
Sep 22 23:22:14.295 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 3 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.295 INFO [0] client ExtentClose { repair_id: ReconciliationId(9), extent_id: 3 }
Sep 22 23:22:14.295 INFO [1] received reconcile message
Sep 22 23:22:14.295 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 3 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.295 INFO [1] client ExtentClose { repair_id: ReconciliationId(9), extent_id: 3 }
Sep 22 23:22:14.295 INFO [2] received reconcile message
Sep 22 23:22:14.295 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(9), op: ExtentClose { repair_id: ReconciliationId(9), extent_id: 3 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.295 INFO [2] client ExtentClose { repair_id: ReconciliationId(9), extent_id: 3 }
Sep 22 23:22:14.295 DEBG 9 Close extent 3
Sep 22 23:22:14.295 DEBG 9 Close extent 3
Sep 22 23:22:14.295 DEBG 9 Close extent 3
Sep 22 23:22:14.296 DEBG [2] It's time to notify for 9
Sep 22 23:22:14.296 INFO Completion from [2] id:9 status:true
Sep 22 23:22:14.296 INFO [10/752] Repair commands completed
Sep 22 23:22:14.296 INFO Pop front: ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.296 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.296 INFO [0] received reconcile message
Sep 22 23:22:14.296 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.296 INFO [0] client ExtentRepair { repair_id: ReconciliationId(10), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.296 INFO [0] Sending repair request ReconciliationId(10)
Sep 22 23:22:14.296 INFO [1] received reconcile message
Sep 22 23:22:14.296 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.296 INFO [1] client ExtentRepair { repair_id: ReconciliationId(10), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.296 INFO [1] No action required ReconciliationId(10)
Sep 22 23:22:14.296 INFO [2] received reconcile message
Sep 22 23:22:14.296 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(10), op: ExtentRepair { repair_id: ReconciliationId(10), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.296 INFO [2] client ExtentRepair { repair_id: ReconciliationId(10), extent_id: 3, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.296 INFO [2] No action required ReconciliationId(10)
Sep 22 23:22:14.296 DEBG 10 Repair extent 3 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.296 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/003.copy"
Sep 22 23:22:14.360 INFO accepted connection, remote_addr: 127.0.0.1:53368, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.360 TRCE incoming request, uri: /extent/3/files, method: GET, req_id: 13b98295-8755-4d86-b0eb-c70f449bc669, remote_addr: 127.0.0.1:53368, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.361 INFO request completed, latency_us: 188, response_code: 200, uri: /extent/3/files, method: GET, req_id: 13b98295-8755-4d86-b0eb-c70f449bc669, remote_addr: 127.0.0.1:53368, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.361 INFO eid:3 Found repair files: ["003", "003.db"]
Sep 22 23:22:14.361 TRCE incoming request, uri: /newextent/3/data, method: GET, req_id: 3ba67e57-64c7-47c7-9d4a-347b27c20bca, remote_addr: 127.0.0.1:53368, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.361 INFO request completed, latency_us: 311, response_code: 200, uri: /newextent/3/data, method: GET, req_id: 3ba67e57-64c7-47c7-9d4a-347b27c20bca, remote_addr: 127.0.0.1:53368, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.366 TRCE incoming request, uri: /newextent/3/db, method: GET, req_id: a9c897a9-cfbd-4957-9df5-5a026f8e251d, remote_addr: 127.0.0.1:53368, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.367 INFO request completed, latency_us: 285, response_code: 200, uri: /newextent/3/db, method: GET, req_id: a9c897a9-cfbd-4957-9df5-5a026f8e251d, remote_addr: 127.0.0.1:53368, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.368 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/003.copy" to "/tmp/downstairs-zrMnlo6G/00/000/003.replace"
Sep 22 23:22:14.368 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.369 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/003.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.369 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/003"
Sep 22 23:22:14.369 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/003.db"
Sep 22 23:22:14.369 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.369 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/003.replace" to "/tmp/downstairs-zrMnlo6G/00/000/003.completed"
Sep 22 23:22:14.369 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.369 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.369 DEBG [0] It's time to notify for 10
Sep 22 23:22:14.369 INFO Completion from [0] id:10 status:true
Sep 22 23:22:14.370 INFO [11/752] Repair commands completed
Sep 22 23:22:14.370 INFO Pop front: ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 3 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.370 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.370 INFO [0] received reconcile message
Sep 22 23:22:14.370 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 3 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.370 INFO [0] client ExtentReopen { repair_id: ReconciliationId(11), extent_id: 3 }
Sep 22 23:22:14.370 INFO [1] received reconcile message
Sep 22 23:22:14.370 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 3 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.370 INFO [1] client ExtentReopen { repair_id: ReconciliationId(11), extent_id: 3 }
Sep 22 23:22:14.370 INFO [2] received reconcile message
Sep 22 23:22:14.370 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(11), op: ExtentReopen { repair_id: ReconciliationId(11), extent_id: 3 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.370 INFO [2] client ExtentReopen { repair_id: ReconciliationId(11), extent_id: 3 }
Sep 22 23:22:14.370 DEBG 11 Reopen extent 3
Sep 22 23:22:14.371 DEBG 11 Reopen extent 3
Sep 22 23:22:14.371 DEBG 11 Reopen extent 3
Sep 22 23:22:14.372 DEBG [2] It's time to notify for 11
Sep 22 23:22:14.372 INFO Completion from [2] id:11 status:true
Sep 22 23:22:14.372 INFO [12/752] Repair commands completed
Sep 22 23:22:14.372 INFO Pop front: ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.372 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.372 INFO [0] received reconcile message
Sep 22 23:22:14.372 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.372 INFO [0] client ExtentFlush { repair_id: ReconciliationId(12), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.372 INFO [1] received reconcile message
Sep 22 23:22:14.372 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.372 INFO [1] client ExtentFlush { repair_id: ReconciliationId(12), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.372 INFO [2] received reconcile message
Sep 22 23:22:14.372 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(12), op: ExtentFlush { repair_id: ReconciliationId(12), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.372 INFO [2] client ExtentFlush { repair_id: ReconciliationId(12), extent_id: 53, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.372 DEBG 12 Flush extent 53 with f:2 g:2
Sep 22 23:22:14.372 DEBG Flush just extent 53 with f:2 and g:2
Sep 22 23:22:14.372 DEBG [1] It's time to notify for 12
Sep 22 23:22:14.372 INFO Completion from [1] id:12 status:true
Sep 22 23:22:14.372 INFO [13/752] Repair commands completed
Sep 22 23:22:14.372 INFO Pop front: ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 53 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.372 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.372 INFO [0] received reconcile message
Sep 22 23:22:14.372 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 53 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.372 INFO [0] client ExtentClose { repair_id: ReconciliationId(13), extent_id: 53 }
Sep 22 23:22:14.372 INFO [1] received reconcile message
Sep 22 23:22:14.372 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 53 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.372 INFO [1] client ExtentClose { repair_id: ReconciliationId(13), extent_id: 53 }
Sep 22 23:22:14.372 INFO [2] received reconcile message
Sep 22 23:22:14.372 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(13), op: ExtentClose { repair_id: ReconciliationId(13), extent_id: 53 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.372 INFO [2] client ExtentClose { repair_id: ReconciliationId(13), extent_id: 53 }
Sep 22 23:22:14.373 DEBG 13 Close extent 53
Sep 22 23:22:14.373 DEBG 13 Close extent 53
Sep 22 23:22:14.373 DEBG 13 Close extent 53
Sep 22 23:22:14.373 DEBG [2] It's time to notify for 13
Sep 22 23:22:14.374 INFO Completion from [2] id:13 status:true
Sep 22 23:22:14.374 INFO [14/752] Repair commands completed
Sep 22 23:22:14.374 INFO Pop front: ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.374 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.374 INFO [0] received reconcile message
Sep 22 23:22:14.374 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.374 INFO [0] client ExtentRepair { repair_id: ReconciliationId(14), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.374 INFO [0] Sending repair request ReconciliationId(14)
Sep 22 23:22:14.374 INFO [1] received reconcile message
Sep 22 23:22:14.374 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.374 INFO [1] client ExtentRepair { repair_id: ReconciliationId(14), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.374 INFO [1] No action required ReconciliationId(14)
Sep 22 23:22:14.374 INFO [2] received reconcile message
Sep 22 23:22:14.374 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(14), op: ExtentRepair { repair_id: ReconciliationId(14), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.374 INFO [2] client ExtentRepair { repair_id: ReconciliationId(14), extent_id: 53, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.374 INFO [2] No action required ReconciliationId(14)
Sep 22 23:22:14.374 DEBG 14 Repair extent 53 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.374 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/035.copy"
Sep 22 23:22:14.438 INFO accepted connection, remote_addr: 127.0.0.1:60377, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.438 TRCE incoming request, uri: /extent/53/files, method: GET, req_id: e2de8886-f68b-48be-8dc0-c8b440a97525, remote_addr: 127.0.0.1:60377, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.438 INFO request completed, latency_us: 207, response_code: 200, uri: /extent/53/files, method: GET, req_id: e2de8886-f68b-48be-8dc0-c8b440a97525, remote_addr: 127.0.0.1:60377, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.438 INFO eid:53 Found repair files: ["035", "035.db"]
Sep 22 23:22:14.439 TRCE incoming request, uri: /newextent/53/data, method: GET, req_id: 8ff8d5b0-6cd8-45ff-958a-196b08d51ab7, remote_addr: 127.0.0.1:60377, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.439 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/53/data, method: GET, req_id: 8ff8d5b0-6cd8-45ff-958a-196b08d51ab7, remote_addr: 127.0.0.1:60377, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.444 TRCE incoming request, uri: /newextent/53/db, method: GET, req_id: 631aa683-1c93-4e4e-a8ec-7f0fa39e7b1c, remote_addr: 127.0.0.1:60377, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.444 INFO request completed, latency_us: 286, response_code: 200, uri: /newextent/53/db, method: GET, req_id: 631aa683-1c93-4e4e-a8ec-7f0fa39e7b1c, remote_addr: 127.0.0.1:60377, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.445 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/035.copy" to "/tmp/downstairs-zrMnlo6G/00/000/035.replace"
Sep 22 23:22:14.445 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.446 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/035.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.446 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/035"
Sep 22 23:22:14.447 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/035.db"
Sep 22 23:22:14.447 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.447 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/035.replace" to "/tmp/downstairs-zrMnlo6G/00/000/035.completed"
Sep 22 23:22:14.447 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.447 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.447 DEBG [0] It's time to notify for 14
Sep 22 23:22:14.447 INFO Completion from [0] id:14 status:true
Sep 22 23:22:14.447 INFO [15/752] Repair commands completed
Sep 22 23:22:14.447 INFO Pop front: ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 53 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.447 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.447 INFO [0] received reconcile message
Sep 22 23:22:14.447 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 53 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.447 INFO [0] client ExtentReopen { repair_id: ReconciliationId(15), extent_id: 53 }
Sep 22 23:22:14.447 INFO [1] received reconcile message
Sep 22 23:22:14.447 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 53 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.447 INFO [1] client ExtentReopen { repair_id: ReconciliationId(15), extent_id: 53 }
Sep 22 23:22:14.447 INFO [2] received reconcile message
Sep 22 23:22:14.447 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(15), op: ExtentReopen { repair_id: ReconciliationId(15), extent_id: 53 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.447 INFO [2] client ExtentReopen { repair_id: ReconciliationId(15), extent_id: 53 }
Sep 22 23:22:14.447 DEBG 15 Reopen extent 53
Sep 22 23:22:14.448 DEBG 15 Reopen extent 53
Sep 22 23:22:14.448 DEBG 15 Reopen extent 53
Sep 22 23:22:14.449 DEBG [2] It's time to notify for 15
Sep 22 23:22:14.449 INFO Completion from [2] id:15 status:true
Sep 22 23:22:14.449 INFO [16/752] Repair commands completed
Sep 22 23:22:14.449 INFO Pop front: ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.449 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.449 INFO [0] received reconcile message
Sep 22 23:22:14.449 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.449 INFO [0] client ExtentFlush { repair_id: ReconciliationId(16), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.449 INFO [1] received reconcile message
Sep 22 23:22:14.449 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.449 INFO [1] client ExtentFlush { repair_id: ReconciliationId(16), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.449 INFO [2] received reconcile message
Sep 22 23:22:14.449 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(16), op: ExtentFlush { repair_id: ReconciliationId(16), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.449 INFO [2] client ExtentFlush { repair_id: ReconciliationId(16), extent_id: 59, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.449 DEBG 16 Flush extent 59 with f:2 g:2
Sep 22 23:22:14.449 DEBG Flush just extent 59 with f:2 and g:2
Sep 22 23:22:14.450 DEBG [1] It's time to notify for 16
Sep 22 23:22:14.450 INFO Completion from [1] id:16 status:true
Sep 22 23:22:14.450 INFO [17/752] Repair commands completed
Sep 22 23:22:14.450 INFO Pop front: ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 59 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.450 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.450 INFO [0] received reconcile message
Sep 22 23:22:14.450 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 59 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.450 INFO [0] client ExtentClose { repair_id: ReconciliationId(17), extent_id: 59 }
Sep 22 23:22:14.450 INFO [1] received reconcile message
Sep 22 23:22:14.450 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 59 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.450 INFO [1] client ExtentClose { repair_id: ReconciliationId(17), extent_id: 59 }
Sep 22 23:22:14.450 INFO [2] received reconcile message
Sep 22 23:22:14.450 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(17), op: ExtentClose { repair_id: ReconciliationId(17), extent_id: 59 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.450 INFO [2] client ExtentClose { repair_id: ReconciliationId(17), extent_id: 59 }
Sep 22 23:22:14.450 DEBG 17 Close extent 59
Sep 22 23:22:14.450 DEBG 17 Close extent 59
Sep 22 23:22:14.451 DEBG 17 Close extent 59
Sep 22 23:22:14.451 DEBG [2] It's time to notify for 17
Sep 22 23:22:14.451 INFO Completion from [2] id:17 status:true
Sep 22 23:22:14.451 INFO [18/752] Repair commands completed
Sep 22 23:22:14.451 INFO Pop front: ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.451 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.451 INFO [0] received reconcile message
Sep 22 23:22:14.451 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.451 INFO [0] client ExtentRepair { repair_id: ReconciliationId(18), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.451 INFO [0] Sending repair request ReconciliationId(18)
Sep 22 23:22:14.451 INFO [1] received reconcile message
Sep 22 23:22:14.451 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.451 INFO [1] client ExtentRepair { repair_id: ReconciliationId(18), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.451 INFO [1] No action required ReconciliationId(18)
Sep 22 23:22:14.451 INFO [2] received reconcile message
Sep 22 23:22:14.451 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(18), op: ExtentRepair { repair_id: ReconciliationId(18), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.451 INFO [2] client ExtentRepair { repair_id: ReconciliationId(18), extent_id: 59, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.451 INFO [2] No action required ReconciliationId(18)
Sep 22 23:22:14.451 DEBG 18 Repair extent 59 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.451 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/03B.copy"
Sep 22 23:22:14.516 INFO accepted connection, remote_addr: 127.0.0.1:43611, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.516 TRCE incoming request, uri: /extent/59/files, method: GET, req_id: cbbee225-db88-4e16-932d-3d40f674ff02, remote_addr: 127.0.0.1:43611, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.516 INFO request completed, latency_us: 187, response_code: 200, uri: /extent/59/files, method: GET, req_id: cbbee225-db88-4e16-932d-3d40f674ff02, remote_addr: 127.0.0.1:43611, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.516 INFO eid:59 Found repair files: ["03B", "03B.db"]
Sep 22 23:22:14.517 TRCE incoming request, uri: /newextent/59/data, method: GET, req_id: b24b7e21-5a24-456b-aee9-769de8ab9a2b, remote_addr: 127.0.0.1:43611, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.517 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/59/data, method: GET, req_id: b24b7e21-5a24-456b-aee9-769de8ab9a2b, remote_addr: 127.0.0.1:43611, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.522 TRCE incoming request, uri: /newextent/59/db, method: GET, req_id: 61251456-69c1-471c-b27a-65d590ed0023, remote_addr: 127.0.0.1:43611, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.522 INFO request completed, latency_us: 289, response_code: 200, uri: /newextent/59/db, method: GET, req_id: 61251456-69c1-471c-b27a-65d590ed0023, remote_addr: 127.0.0.1:43611, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.523 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/03B.copy" to "/tmp/downstairs-zrMnlo6G/00/000/03B.replace"
Sep 22 23:22:14.523 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.524 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/03B.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.525 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03B"
Sep 22 23:22:14.525 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03B.db"
Sep 22 23:22:14.525 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.525 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/03B.replace" to "/tmp/downstairs-zrMnlo6G/00/000/03B.completed"
Sep 22 23:22:14.525 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.525 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.525 DEBG [0] It's time to notify for 18
Sep 22 23:22:14.525 INFO Completion from [0] id:18 status:true
Sep 22 23:22:14.525 INFO [19/752] Repair commands completed
Sep 22 23:22:14.525 INFO Pop front: ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 59 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.525 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.525 INFO [0] received reconcile message
Sep 22 23:22:14.525 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 59 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.525 INFO [0] client ExtentReopen { repair_id: ReconciliationId(19), extent_id: 59 }
Sep 22 23:22:14.525 INFO [1] received reconcile message
Sep 22 23:22:14.525 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 59 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.525 INFO [1] client ExtentReopen { repair_id: ReconciliationId(19), extent_id: 59 }
Sep 22 23:22:14.525 INFO [2] received reconcile message
Sep 22 23:22:14.525 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(19), op: ExtentReopen { repair_id: ReconciliationId(19), extent_id: 59 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.525 INFO [2] client ExtentReopen { repair_id: ReconciliationId(19), extent_id: 59 }
Sep 22 23:22:14.525 DEBG 19 Reopen extent 59
Sep 22 23:22:14.526 DEBG 19 Reopen extent 59
Sep 22 23:22:14.526 DEBG 19 Reopen extent 59
Sep 22 23:22:14.527 DEBG [2] It's time to notify for 19
Sep 22 23:22:14.527 INFO Completion from [2] id:19 status:true
Sep 22 23:22:14.527 INFO [20/752] Repair commands completed
Sep 22 23:22:14.527 INFO Pop front: ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.527 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.527 INFO [0] received reconcile message
Sep 22 23:22:14.527 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.527 INFO [0] client ExtentFlush { repair_id: ReconciliationId(20), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.527 INFO [1] received reconcile message
Sep 22 23:22:14.527 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.527 INFO [1] client ExtentFlush { repair_id: ReconciliationId(20), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.527 INFO [2] received reconcile message
Sep 22 23:22:14.527 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(20), op: ExtentFlush { repair_id: ReconciliationId(20), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.527 INFO [2] client ExtentFlush { repair_id: ReconciliationId(20), extent_id: 103, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.527 DEBG 20 Flush extent 103 with f:2 g:2
Sep 22 23:22:14.527 DEBG Flush just extent 103 with f:2 and g:2
Sep 22 23:22:14.528 DEBG [1] It's time to notify for 20
Sep 22 23:22:14.528 INFO Completion from [1] id:20 status:true
Sep 22 23:22:14.528 INFO [21/752] Repair commands completed
Sep 22 23:22:14.528 INFO Pop front: ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 103 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.528 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.528 INFO [0] received reconcile message
Sep 22 23:22:14.528 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 103 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.528 INFO [0] client ExtentClose { repair_id: ReconciliationId(21), extent_id: 103 }
Sep 22 23:22:14.528 INFO [1] received reconcile message
Sep 22 23:22:14.528 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 103 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.528 INFO [1] client ExtentClose { repair_id: ReconciliationId(21), extent_id: 103 }
Sep 22 23:22:14.528 INFO [2] received reconcile message
Sep 22 23:22:14.528 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(21), op: ExtentClose { repair_id: ReconciliationId(21), extent_id: 103 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.528 INFO [2] client ExtentClose { repair_id: ReconciliationId(21), extent_id: 103 }
Sep 22 23:22:14.528 DEBG 21 Close extent 103
Sep 22 23:22:14.528 DEBG 21 Close extent 103
Sep 22 23:22:14.529 DEBG 21 Close extent 103
Sep 22 23:22:14.529 DEBG [2] It's time to notify for 21
Sep 22 23:22:14.529 INFO Completion from [2] id:21 status:true
Sep 22 23:22:14.529 INFO [22/752] Repair commands completed
Sep 22 23:22:14.529 INFO Pop front: ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.529 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.529 INFO [0] received reconcile message
Sep 22 23:22:14.529 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.529 INFO [0] client ExtentRepair { repair_id: ReconciliationId(22), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.529 INFO [0] Sending repair request ReconciliationId(22)
Sep 22 23:22:14.529 INFO [1] received reconcile message
Sep 22 23:22:14.529 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.529 INFO [1] client ExtentRepair { repair_id: ReconciliationId(22), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.529 INFO [1] No action required ReconciliationId(22)
Sep 22 23:22:14.529 INFO [2] received reconcile message
Sep 22 23:22:14.529 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(22), op: ExtentRepair { repair_id: ReconciliationId(22), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.529 INFO [2] client ExtentRepair { repair_id: ReconciliationId(22), extent_id: 103, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.529 INFO [2] No action required ReconciliationId(22)
Sep 22 23:22:14.529 DEBG 22 Repair extent 103 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.529 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/067.copy"
Sep 22 23:22:14.593 INFO accepted connection, remote_addr: 127.0.0.1:48311, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.594 TRCE incoming request, uri: /extent/103/files, method: GET, req_id: 9f723176-0cd4-4975-9695-22e5b3e4c325, remote_addr: 127.0.0.1:48311, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.594 INFO request completed, latency_us: 189, response_code: 200, uri: /extent/103/files, method: GET, req_id: 9f723176-0cd4-4975-9695-22e5b3e4c325, remote_addr: 127.0.0.1:48311, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.594 INFO eid:103 Found repair files: ["067", "067.db"]
Sep 22 23:22:14.594 TRCE incoming request, uri: /newextent/103/data, method: GET, req_id: 038f18a2-8a71-4da7-853d-ab8848aaa44e, remote_addr: 127.0.0.1:48311, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.595 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/103/data, method: GET, req_id: 038f18a2-8a71-4da7-853d-ab8848aaa44e, remote_addr: 127.0.0.1:48311, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.600 TRCE incoming request, uri: /newextent/103/db, method: GET, req_id: afd79e50-7d2b-40e7-8236-3811abe6ec64, remote_addr: 127.0.0.1:48311, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.600 INFO request completed, latency_us: 288, response_code: 200, uri: /newextent/103/db, method: GET, req_id: afd79e50-7d2b-40e7-8236-3811abe6ec64, remote_addr: 127.0.0.1:48311, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.601 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/067.copy" to "/tmp/downstairs-zrMnlo6G/00/000/067.replace"
Sep 22 23:22:14.601 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.602 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/067.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.602 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/067"
Sep 22 23:22:14.602 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/067.db"
Sep 22 23:22:14.602 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.602 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/067.replace" to "/tmp/downstairs-zrMnlo6G/00/000/067.completed"
Sep 22 23:22:14.602 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.602 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.603 DEBG [0] It's time to notify for 22
Sep 22 23:22:14.603 INFO Completion from [0] id:22 status:true
Sep 22 23:22:14.603 INFO [23/752] Repair commands completed
Sep 22 23:22:14.603 INFO Pop front: ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 103 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.603 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.603 INFO [0] received reconcile message
Sep 22 23:22:14.603 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 103 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.603 INFO [0] client ExtentReopen { repair_id: ReconciliationId(23), extent_id: 103 }
Sep 22 23:22:14.603 INFO [1] received reconcile message
Sep 22 23:22:14.603 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 103 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.603 INFO [1] client ExtentReopen { repair_id: ReconciliationId(23), extent_id: 103 }
Sep 22 23:22:14.603 INFO [2] received reconcile message
Sep 22 23:22:14.603 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(23), op: ExtentReopen { repair_id: ReconciliationId(23), extent_id: 103 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.603 INFO [2] client ExtentReopen { repair_id: ReconciliationId(23), extent_id: 103 }
Sep 22 23:22:14.603 DEBG 23 Reopen extent 103
Sep 22 23:22:14.604 DEBG 23 Reopen extent 103
Sep 22 23:22:14.604 DEBG 23 Reopen extent 103
Sep 22 23:22:14.605 DEBG [2] It's time to notify for 23
Sep 22 23:22:14.605 INFO Completion from [2] id:23 status:true
Sep 22 23:22:14.605 INFO [24/752] Repair commands completed
Sep 22 23:22:14.605 INFO Pop front: ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.605 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.605 INFO [0] received reconcile message
Sep 22 23:22:14.605 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.605 INFO [0] client ExtentFlush { repair_id: ReconciliationId(24), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.605 INFO [1] received reconcile message
Sep 22 23:22:14.605 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.605 INFO [1] client ExtentFlush { repair_id: ReconciliationId(24), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.605 INFO [2] received reconcile message
Sep 22 23:22:14.605 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(24), op: ExtentFlush { repair_id: ReconciliationId(24), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.605 INFO [2] client ExtentFlush { repair_id: ReconciliationId(24), extent_id: 92, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.605 DEBG 24 Flush extent 92 with f:2 g:2
Sep 22 23:22:14.605 DEBG Flush just extent 92 with f:2 and g:2
Sep 22 23:22:14.605 DEBG [1] It's time to notify for 24
Sep 22 23:22:14.605 INFO Completion from [1] id:24 status:true
Sep 22 23:22:14.605 INFO [25/752] Repair commands completed
Sep 22 23:22:14.605 INFO Pop front: ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 92 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.605 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.605 INFO [0] received reconcile message
Sep 22 23:22:14.605 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 92 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.605 INFO [0] client ExtentClose { repair_id: ReconciliationId(25), extent_id: 92 }
Sep 22 23:22:14.606 INFO [1] received reconcile message
Sep 22 23:22:14.606 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 92 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.606 INFO [1] client ExtentClose { repair_id: ReconciliationId(25), extent_id: 92 }
Sep 22 23:22:14.606 INFO [2] received reconcile message
Sep 22 23:22:14.606 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(25), op: ExtentClose { repair_id: ReconciliationId(25), extent_id: 92 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.606 INFO [2] client ExtentClose { repair_id: ReconciliationId(25), extent_id: 92 }
Sep 22 23:22:14.606 DEBG 25 Close extent 92
Sep 22 23:22:14.606 DEBG 25 Close extent 92
Sep 22 23:22:14.606 DEBG 25 Close extent 92
Sep 22 23:22:14.607 DEBG [2] It's time to notify for 25
Sep 22 23:22:14.607 INFO Completion from [2] id:25 status:true
Sep 22 23:22:14.607 INFO [26/752] Repair commands completed
Sep 22 23:22:14.607 INFO Pop front: ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.607 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.607 INFO [0] received reconcile message
Sep 22 23:22:14.607 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.607 INFO [0] client ExtentRepair { repair_id: ReconciliationId(26), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.607 INFO [0] Sending repair request ReconciliationId(26)
Sep 22 23:22:14.607 INFO [1] received reconcile message
Sep 22 23:22:14.607 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.607 INFO [1] client ExtentRepair { repair_id: ReconciliationId(26), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.607 INFO [1] No action required ReconciliationId(26)
Sep 22 23:22:14.607 INFO [2] received reconcile message
Sep 22 23:22:14.607 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(26), op: ExtentRepair { repair_id: ReconciliationId(26), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.607 INFO [2] client ExtentRepair { repair_id: ReconciliationId(26), extent_id: 92, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.607 INFO [2] No action required ReconciliationId(26)
Sep 22 23:22:14.607 DEBG 26 Repair extent 92 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.607 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/05C.copy"
Sep 22 23:22:14.671 INFO accepted connection, remote_addr: 127.0.0.1:50918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.671 TRCE incoming request, uri: /extent/92/files, method: GET, req_id: 620a61dd-2d27-48e0-b04f-846e91ee0976, remote_addr: 127.0.0.1:50918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.671 INFO request completed, latency_us: 197, response_code: 200, uri: /extent/92/files, method: GET, req_id: 620a61dd-2d27-48e0-b04f-846e91ee0976, remote_addr: 127.0.0.1:50918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.671 INFO eid:92 Found repair files: ["05C", "05C.db"]
Sep 22 23:22:14.672 TRCE incoming request, uri: /newextent/92/data, method: GET, req_id: 1663aa58-ba20-4776-9956-e0571654aa78, remote_addr: 127.0.0.1:50918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.672 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/92/data, method: GET, req_id: 1663aa58-ba20-4776-9956-e0571654aa78, remote_addr: 127.0.0.1:50918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.677 TRCE incoming request, uri: /newextent/92/db, method: GET, req_id: 7d6f3054-7044-4270-bd00-f78c6fe8e527, remote_addr: 127.0.0.1:50918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.677 INFO request completed, latency_us: 287, response_code: 200, uri: /newextent/92/db, method: GET, req_id: 7d6f3054-7044-4270-bd00-f78c6fe8e527, remote_addr: 127.0.0.1:50918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.678 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/05C.copy" to "/tmp/downstairs-zrMnlo6G/00/000/05C.replace"
Sep 22 23:22:14.679 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.679 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/05C.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.680 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05C"
Sep 22 23:22:14.680 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05C.db"
Sep 22 23:22:14.680 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.680 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/05C.replace" to "/tmp/downstairs-zrMnlo6G/00/000/05C.completed"
Sep 22 23:22:14.680 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.680 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.680 DEBG [0] It's time to notify for 26
Sep 22 23:22:14.680 INFO Completion from [0] id:26 status:true
Sep 22 23:22:14.680 INFO [27/752] Repair commands completed
Sep 22 23:22:14.680 INFO Pop front: ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 92 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.680 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.680 INFO [0] received reconcile message
Sep 22 23:22:14.680 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 92 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.680 INFO [0] client ExtentReopen { repair_id: ReconciliationId(27), extent_id: 92 }
Sep 22 23:22:14.680 INFO [1] received reconcile message
Sep 22 23:22:14.680 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 92 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.680 INFO [1] client ExtentReopen { repair_id: ReconciliationId(27), extent_id: 92 }
Sep 22 23:22:14.680 INFO [2] received reconcile message
Sep 22 23:22:14.680 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(27), op: ExtentReopen { repair_id: ReconciliationId(27), extent_id: 92 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.680 INFO [2] client ExtentReopen { repair_id: ReconciliationId(27), extent_id: 92 }
Sep 22 23:22:14.680 DEBG 27 Reopen extent 92
Sep 22 23:22:14.681 DEBG 27 Reopen extent 92
Sep 22 23:22:14.682 DEBG 27 Reopen extent 92
Sep 22 23:22:14.682 DEBG [2] It's time to notify for 27
Sep 22 23:22:14.682 INFO Completion from [2] id:27 status:true
Sep 22 23:22:14.682 INFO [28/752] Repair commands completed
Sep 22 23:22:14.682 INFO Pop front: ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.682 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.682 INFO [0] received reconcile message
Sep 22 23:22:14.682 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.682 INFO [0] client ExtentFlush { repair_id: ReconciliationId(28), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.682 INFO [1] received reconcile message
Sep 22 23:22:14.682 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.682 INFO [1] client ExtentFlush { repair_id: ReconciliationId(28), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.682 INFO [2] received reconcile message
Sep 22 23:22:14.682 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(28), op: ExtentFlush { repair_id: ReconciliationId(28), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.682 INFO [2] client ExtentFlush { repair_id: ReconciliationId(28), extent_id: 129, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.682 DEBG 28 Flush extent 129 with f:2 g:2
Sep 22 23:22:14.683 DEBG Flush just extent 129 with f:2 and g:2
Sep 22 23:22:14.683 DEBG [1] It's time to notify for 28
Sep 22 23:22:14.683 INFO Completion from [1] id:28 status:true
Sep 22 23:22:14.683 INFO [29/752] Repair commands completed
Sep 22 23:22:14.683 INFO Pop front: ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 129 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.683 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.683 INFO [0] received reconcile message
Sep 22 23:22:14.683 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 129 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.683 INFO [0] client ExtentClose { repair_id: ReconciliationId(29), extent_id: 129 }
Sep 22 23:22:14.683 INFO [1] received reconcile message
Sep 22 23:22:14.683 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 129 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.683 INFO [1] client ExtentClose { repair_id: ReconciliationId(29), extent_id: 129 }
Sep 22 23:22:14.683 INFO [2] received reconcile message
Sep 22 23:22:14.683 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(29), op: ExtentClose { repair_id: ReconciliationId(29), extent_id: 129 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.683 INFO [2] client ExtentClose { repair_id: ReconciliationId(29), extent_id: 129 }
Sep 22 23:22:14.683 DEBG 29 Close extent 129
Sep 22 23:22:14.683 DEBG 29 Close extent 129
Sep 22 23:22:14.684 DEBG 29 Close extent 129
Sep 22 23:22:14.684 DEBG [2] It's time to notify for 29
Sep 22 23:22:14.684 INFO Completion from [2] id:29 status:true
Sep 22 23:22:14.684 INFO [30/752] Repair commands completed
Sep 22 23:22:14.684 INFO Pop front: ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.684 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.684 INFO [0] received reconcile message
Sep 22 23:22:14.684 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.684 INFO [0] client ExtentRepair { repair_id: ReconciliationId(30), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.684 INFO [0] Sending repair request ReconciliationId(30)
Sep 22 23:22:14.684 INFO [1] received reconcile message
Sep 22 23:22:14.684 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.684 INFO [1] client ExtentRepair { repair_id: ReconciliationId(30), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.684 INFO [1] No action required ReconciliationId(30)
Sep 22 23:22:14.684 INFO [2] received reconcile message
Sep 22 23:22:14.684 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(30), op: ExtentRepair { repair_id: ReconciliationId(30), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.684 INFO [2] client ExtentRepair { repair_id: ReconciliationId(30), extent_id: 129, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.684 INFO [2] No action required ReconciliationId(30)
Sep 22 23:22:14.684 DEBG 30 Repair extent 129 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.685 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/081.copy"
Sep 22 23:22:14.705 DEBG up_ds_listen was notified
Sep 22 23:22:14.705 DEBG up_ds_listen process 1072
Sep 22 23:22:14.705 DEBG [A] ack job 1072:73, : downstairs
Sep 22 23:22:14.705 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:14.705 DEBG IO Flush 1074 has deps [JobId(1073), JobId(1072)]
Sep 22 23:22:14.705 WARN returning error on read!
Sep 22 23:22:14.705 DEBG Read      :1073 deps:[JobId(1072)] res:false
Sep 22 23:22:14.711 DEBG Read      :1073 deps:[JobId(1072)] res:true
Sep 22 23:22:14.733 ERRO [0] job id 1073 saw error GenericError("test error")
Sep 22 23:22:14.735 DEBG Flush     :1074 extent_limit None deps:[JobId(1073), JobId(1072)] res:true f:26 g:1
Sep 22 23:22:14.735 INFO [lossy] sleeping 1 second
Sep 22 23:22:14.748 INFO accepted connection, remote_addr: 127.0.0.1:65069, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.749 TRCE incoming request, uri: /extent/129/files, method: GET, req_id: 7f3661a4-53bd-4465-8f89-127d245e5b6e, remote_addr: 127.0.0.1:65069, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.749 INFO request completed, latency_us: 189, response_code: 200, uri: /extent/129/files, method: GET, req_id: 7f3661a4-53bd-4465-8f89-127d245e5b6e, remote_addr: 127.0.0.1:65069, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.749 INFO eid:129 Found repair files: ["081", "081.db"]
Sep 22 23:22:14.749 TRCE incoming request, uri: /newextent/129/data, method: GET, req_id: 867aca05-4a93-4317-bb7a-b02e9e74a39e, remote_addr: 127.0.0.1:65069, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.750 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/129/data, method: GET, req_id: 867aca05-4a93-4317-bb7a-b02e9e74a39e, remote_addr: 127.0.0.1:65069, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.755 TRCE incoming request, uri: /newextent/129/db, method: GET, req_id: 450b3e0d-5671-440a-a2a5-ecde4274eb0e, remote_addr: 127.0.0.1:65069, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.755 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/129/db, method: GET, req_id: 450b3e0d-5671-440a-a2a5-ecde4274eb0e, remote_addr: 127.0.0.1:65069, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.756 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/081.copy" to "/tmp/downstairs-zrMnlo6G/00/000/081.replace"
Sep 22 23:22:14.756 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.757 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/081.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.757 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/081"
Sep 22 23:22:14.757 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/081.db"
Sep 22 23:22:14.757 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.757 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/081.replace" to "/tmp/downstairs-zrMnlo6G/00/000/081.completed"
Sep 22 23:22:14.757 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.757 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.758 DEBG [0] It's time to notify for 30
Sep 22 23:22:14.758 INFO Completion from [0] id:30 status:true
Sep 22 23:22:14.758 INFO [31/752] Repair commands completed
Sep 22 23:22:14.758 INFO Pop front: ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 129 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.758 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.758 INFO [0] received reconcile message
Sep 22 23:22:14.758 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 129 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.758 INFO [0] client ExtentReopen { repair_id: ReconciliationId(31), extent_id: 129 }
Sep 22 23:22:14.758 INFO [1] received reconcile message
Sep 22 23:22:14.758 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 129 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.758 INFO [1] client ExtentReopen { repair_id: ReconciliationId(31), extent_id: 129 }
Sep 22 23:22:14.758 INFO [2] received reconcile message
Sep 22 23:22:14.758 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(31), op: ExtentReopen { repair_id: ReconciliationId(31), extent_id: 129 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.758 INFO [2] client ExtentReopen { repair_id: ReconciliationId(31), extent_id: 129 }
Sep 22 23:22:14.758 DEBG 31 Reopen extent 129
Sep 22 23:22:14.759 DEBG 31 Reopen extent 129
Sep 22 23:22:14.759 DEBG 31 Reopen extent 129
Sep 22 23:22:14.760 DEBG [2] It's time to notify for 31
Sep 22 23:22:14.760 INFO Completion from [2] id:31 status:true
Sep 22 23:22:14.760 INFO [32/752] Repair commands completed
Sep 22 23:22:14.760 INFO Pop front: ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.760 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.760 INFO [0] received reconcile message
Sep 22 23:22:14.760 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.760 INFO [0] client ExtentFlush { repair_id: ReconciliationId(32), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.760 INFO [1] received reconcile message
Sep 22 23:22:14.760 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.760 INFO [1] client ExtentFlush { repair_id: ReconciliationId(32), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.760 INFO [2] received reconcile message
Sep 22 23:22:14.760 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(32), op: ExtentFlush { repair_id: ReconciliationId(32), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.760 INFO [2] client ExtentFlush { repair_id: ReconciliationId(32), extent_id: 132, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.760 DEBG 32 Flush extent 132 with f:2 g:2
Sep 22 23:22:14.760 DEBG Flush just extent 132 with f:2 and g:2
Sep 22 23:22:14.760 DEBG [1] It's time to notify for 32
Sep 22 23:22:14.760 INFO Completion from [1] id:32 status:true
Sep 22 23:22:14.760 INFO [33/752] Repair commands completed
Sep 22 23:22:14.760 INFO Pop front: ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 132 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.760 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.760 INFO [0] received reconcile message
Sep 22 23:22:14.760 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 132 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.760 INFO [0] client ExtentClose { repair_id: ReconciliationId(33), extent_id: 132 }
Sep 22 23:22:14.761 INFO [1] received reconcile message
Sep 22 23:22:14.761 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 132 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.761 INFO [1] client ExtentClose { repair_id: ReconciliationId(33), extent_id: 132 }
Sep 22 23:22:14.761 INFO [2] received reconcile message
Sep 22 23:22:14.761 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(33), op: ExtentClose { repair_id: ReconciliationId(33), extent_id: 132 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.761 INFO [2] client ExtentClose { repair_id: ReconciliationId(33), extent_id: 132 }
Sep 22 23:22:14.761 DEBG 33 Close extent 132
Sep 22 23:22:14.761 DEBG 33 Close extent 132
Sep 22 23:22:14.761 DEBG 33 Close extent 132
Sep 22 23:22:14.762 DEBG [2] It's time to notify for 33
Sep 22 23:22:14.762 INFO Completion from [2] id:33 status:true
Sep 22 23:22:14.762 INFO [34/752] Repair commands completed
Sep 22 23:22:14.762 INFO Pop front: ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.762 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.762 INFO [0] received reconcile message
Sep 22 23:22:14.762 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.762 INFO [0] client ExtentRepair { repair_id: ReconciliationId(34), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.762 INFO [0] Sending repair request ReconciliationId(34)
Sep 22 23:22:14.762 INFO [1] received reconcile message
Sep 22 23:22:14.762 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.762 INFO [1] client ExtentRepair { repair_id: ReconciliationId(34), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.762 INFO [1] No action required ReconciliationId(34)
Sep 22 23:22:14.762 INFO [2] received reconcile message
Sep 22 23:22:14.762 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(34), op: ExtentRepair { repair_id: ReconciliationId(34), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.762 INFO [2] client ExtentRepair { repair_id: ReconciliationId(34), extent_id: 132, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.762 INFO [2] No action required ReconciliationId(34)
Sep 22 23:22:14.762 DEBG 34 Repair extent 132 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.762 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/084.copy"
Sep 22 23:22:14.826 INFO accepted connection, remote_addr: 127.0.0.1:47553, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.826 TRCE incoming request, uri: /extent/132/files, method: GET, req_id: 2e4c41b8-7d13-464c-b488-39823517979d, remote_addr: 127.0.0.1:47553, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.826 INFO request completed, latency_us: 211, response_code: 200, uri: /extent/132/files, method: GET, req_id: 2e4c41b8-7d13-464c-b488-39823517979d, remote_addr: 127.0.0.1:47553, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.826 INFO eid:132 Found repair files: ["084", "084.db"]
Sep 22 23:22:14.827 TRCE incoming request, uri: /newextent/132/data, method: GET, req_id: df772583-ee58-4a3b-bd82-f2580c4d8093, remote_addr: 127.0.0.1:47553, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.827 INFO request completed, latency_us: 319, response_code: 200, uri: /newextent/132/data, method: GET, req_id: df772583-ee58-4a3b-bd82-f2580c4d8093, remote_addr: 127.0.0.1:47553, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.832 TRCE incoming request, uri: /newextent/132/db, method: GET, req_id: dd0bedee-1108-41b3-8516-4e7c351ee3e0, remote_addr: 127.0.0.1:47553, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.832 INFO request completed, latency_us: 289, response_code: 200, uri: /newextent/132/db, method: GET, req_id: dd0bedee-1108-41b3-8516-4e7c351ee3e0, remote_addr: 127.0.0.1:47553, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.833 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/084.copy" to "/tmp/downstairs-zrMnlo6G/00/000/084.replace"
Sep 22 23:22:14.834 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.834 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/084.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.835 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/084"
Sep 22 23:22:14.835 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/084.db"
Sep 22 23:22:14.835 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.835 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/084.replace" to "/tmp/downstairs-zrMnlo6G/00/000/084.completed"
Sep 22 23:22:14.835 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.835 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.835 DEBG [0] It's time to notify for 34
Sep 22 23:22:14.835 INFO Completion from [0] id:34 status:true
Sep 22 23:22:14.835 INFO [35/752] Repair commands completed
Sep 22 23:22:14.835 INFO Pop front: ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 132 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.835 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.835 INFO [0] received reconcile message
Sep 22 23:22:14.835 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 132 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.835 INFO [0] client ExtentReopen { repair_id: ReconciliationId(35), extent_id: 132 }
Sep 22 23:22:14.835 INFO [1] received reconcile message
Sep 22 23:22:14.835 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 132 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.835 INFO [1] client ExtentReopen { repair_id: ReconciliationId(35), extent_id: 132 }
Sep 22 23:22:14.835 INFO [2] received reconcile message
Sep 22 23:22:14.835 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(35), op: ExtentReopen { repair_id: ReconciliationId(35), extent_id: 132 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.835 INFO [2] client ExtentReopen { repair_id: ReconciliationId(35), extent_id: 132 }
Sep 22 23:22:14.836 DEBG 35 Reopen extent 132
Sep 22 23:22:14.836 DEBG 35 Reopen extent 132
Sep 22 23:22:14.837 DEBG 35 Reopen extent 132
Sep 22 23:22:14.837 DEBG [2] It's time to notify for 35
Sep 22 23:22:14.837 INFO Completion from [2] id:35 status:true
Sep 22 23:22:14.837 INFO [36/752] Repair commands completed
Sep 22 23:22:14.837 INFO Pop front: ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.837 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.837 INFO [0] received reconcile message
Sep 22 23:22:14.837 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.837 INFO [0] client ExtentFlush { repair_id: ReconciliationId(36), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.838 INFO [1] received reconcile message
Sep 22 23:22:14.838 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.838 INFO [1] client ExtentFlush { repair_id: ReconciliationId(36), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.838 INFO [2] received reconcile message
Sep 22 23:22:14.838 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(36), op: ExtentFlush { repair_id: ReconciliationId(36), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.838 INFO [2] client ExtentFlush { repair_id: ReconciliationId(36), extent_id: 138, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.838 DEBG 36 Flush extent 138 with f:2 g:2
Sep 22 23:22:14.838 DEBG Flush just extent 138 with f:2 and g:2
Sep 22 23:22:14.838 DEBG [1] It's time to notify for 36
Sep 22 23:22:14.838 INFO Completion from [1] id:36 status:true
Sep 22 23:22:14.838 INFO [37/752] Repair commands completed
Sep 22 23:22:14.838 INFO Pop front: ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 138 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.838 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.838 INFO [0] received reconcile message
Sep 22 23:22:14.838 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 138 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.838 INFO [0] client ExtentClose { repair_id: ReconciliationId(37), extent_id: 138 }
Sep 22 23:22:14.838 INFO [1] received reconcile message
Sep 22 23:22:14.838 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 138 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.838 INFO [1] client ExtentClose { repair_id: ReconciliationId(37), extent_id: 138 }
Sep 22 23:22:14.838 INFO [2] received reconcile message
Sep 22 23:22:14.838 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(37), op: ExtentClose { repair_id: ReconciliationId(37), extent_id: 138 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.838 INFO [2] client ExtentClose { repair_id: ReconciliationId(37), extent_id: 138 }
Sep 22 23:22:14.838 DEBG 37 Close extent 138
Sep 22 23:22:14.839 DEBG 37 Close extent 138
Sep 22 23:22:14.839 DEBG 37 Close extent 138
Sep 22 23:22:14.839 DEBG [2] It's time to notify for 37
Sep 22 23:22:14.839 INFO Completion from [2] id:37 status:true
Sep 22 23:22:14.839 INFO [38/752] Repair commands completed
Sep 22 23:22:14.839 INFO Pop front: ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.839 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.839 INFO [0] received reconcile message
Sep 22 23:22:14.839 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.839 INFO [0] client ExtentRepair { repair_id: ReconciliationId(38), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.839 INFO [0] Sending repair request ReconciliationId(38)
Sep 22 23:22:14.840 INFO [1] received reconcile message
Sep 22 23:22:14.840 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.840 INFO [1] client ExtentRepair { repair_id: ReconciliationId(38), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.840 INFO [1] No action required ReconciliationId(38)
Sep 22 23:22:14.840 INFO [2] received reconcile message
Sep 22 23:22:14.840 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(38), op: ExtentRepair { repair_id: ReconciliationId(38), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.840 INFO [2] client ExtentRepair { repair_id: ReconciliationId(38), extent_id: 138, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.840 INFO [2] No action required ReconciliationId(38)
Sep 22 23:22:14.840 DEBG 38 Repair extent 138 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.840 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/08A.copy"
Sep 22 23:22:14.903 INFO accepted connection, remote_addr: 127.0.0.1:45746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.903 TRCE incoming request, uri: /extent/138/files, method: GET, req_id: 82e356fb-170d-41e3-90b8-0c3335940202, remote_addr: 127.0.0.1:45746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.903 INFO request completed, latency_us: 195, response_code: 200, uri: /extent/138/files, method: GET, req_id: 82e356fb-170d-41e3-90b8-0c3335940202, remote_addr: 127.0.0.1:45746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.904 INFO eid:138 Found repair files: ["08A", "08A.db"]
Sep 22 23:22:14.904 TRCE incoming request, uri: /newextent/138/data, method: GET, req_id: 23032a33-1dfc-4476-8b8b-3d20ae4b259f, remote_addr: 127.0.0.1:45746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.904 INFO request completed, latency_us: 313, response_code: 200, uri: /newextent/138/data, method: GET, req_id: 23032a33-1dfc-4476-8b8b-3d20ae4b259f, remote_addr: 127.0.0.1:45746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.909 TRCE incoming request, uri: /newextent/138/db, method: GET, req_id: 859b53c4-b38e-4f40-a8b4-6e1506c81b5d, remote_addr: 127.0.0.1:45746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.910 INFO request completed, latency_us: 288, response_code: 200, uri: /newextent/138/db, method: GET, req_id: 859b53c4-b38e-4f40-a8b4-6e1506c81b5d, remote_addr: 127.0.0.1:45746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.911 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/08A.copy" to "/tmp/downstairs-zrMnlo6G/00/000/08A.replace"
Sep 22 23:22:14.911 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.912 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/08A.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.912 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08A"
Sep 22 23:22:14.912 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08A.db"
Sep 22 23:22:14.912 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.912 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/08A.replace" to "/tmp/downstairs-zrMnlo6G/00/000/08A.completed"
Sep 22 23:22:14.912 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.912 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.912 DEBG [0] It's time to notify for 38
Sep 22 23:22:14.912 INFO Completion from [0] id:38 status:true
Sep 22 23:22:14.912 INFO [39/752] Repair commands completed
Sep 22 23:22:14.912 INFO Pop front: ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 138 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.913 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.913 INFO [0] received reconcile message
Sep 22 23:22:14.913 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 138 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.913 INFO [0] client ExtentReopen { repair_id: ReconciliationId(39), extent_id: 138 }
Sep 22 23:22:14.913 INFO [1] received reconcile message
Sep 22 23:22:14.913 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 138 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.913 INFO [1] client ExtentReopen { repair_id: ReconciliationId(39), extent_id: 138 }
Sep 22 23:22:14.913 INFO [2] received reconcile message
Sep 22 23:22:14.913 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(39), op: ExtentReopen { repair_id: ReconciliationId(39), extent_id: 138 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.913 INFO [2] client ExtentReopen { repair_id: ReconciliationId(39), extent_id: 138 }
Sep 22 23:22:14.913 DEBG 39 Reopen extent 138
Sep 22 23:22:14.913 DEBG 39 Reopen extent 138
Sep 22 23:22:14.914 DEBG 39 Reopen extent 138
Sep 22 23:22:14.914 DEBG [2] It's time to notify for 39
Sep 22 23:22:14.915 INFO Completion from [2] id:39 status:true
Sep 22 23:22:14.915 INFO [40/752] Repair commands completed
Sep 22 23:22:14.915 INFO Pop front: ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.915 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.915 INFO [0] received reconcile message
Sep 22 23:22:14.915 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.915 INFO [0] client ExtentFlush { repair_id: ReconciliationId(40), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.915 INFO [1] received reconcile message
Sep 22 23:22:14.915 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.915 INFO [1] client ExtentFlush { repair_id: ReconciliationId(40), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.915 INFO [2] received reconcile message
Sep 22 23:22:14.915 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(40), op: ExtentFlush { repair_id: ReconciliationId(40), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.915 INFO [2] client ExtentFlush { repair_id: ReconciliationId(40), extent_id: 147, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.915 DEBG 40 Flush extent 147 with f:2 g:2
Sep 22 23:22:14.915 DEBG Flush just extent 147 with f:2 and g:2
Sep 22 23:22:14.915 DEBG [1] It's time to notify for 40
Sep 22 23:22:14.915 INFO Completion from [1] id:40 status:true
Sep 22 23:22:14.915 INFO [41/752] Repair commands completed
Sep 22 23:22:14.915 INFO Pop front: ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 147 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.915 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.915 INFO [0] received reconcile message
Sep 22 23:22:14.915 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 147 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.915 INFO [0] client ExtentClose { repair_id: ReconciliationId(41), extent_id: 147 }
Sep 22 23:22:14.915 INFO [1] received reconcile message
Sep 22 23:22:14.915 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 147 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.915 INFO [1] client ExtentClose { repair_id: ReconciliationId(41), extent_id: 147 }
Sep 22 23:22:14.915 INFO [2] received reconcile message
Sep 22 23:22:14.915 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(41), op: ExtentClose { repair_id: ReconciliationId(41), extent_id: 147 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.915 INFO [2] client ExtentClose { repair_id: ReconciliationId(41), extent_id: 147 }
Sep 22 23:22:14.916 DEBG 41 Close extent 147
Sep 22 23:22:14.916 DEBG 41 Close extent 147
Sep 22 23:22:14.916 DEBG 41 Close extent 147
Sep 22 23:22:14.916 DEBG [2] It's time to notify for 41
Sep 22 23:22:14.916 INFO Completion from [2] id:41 status:true
Sep 22 23:22:14.916 INFO [42/752] Repair commands completed
Sep 22 23:22:14.917 INFO Pop front: ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.917 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.917 INFO [0] received reconcile message
Sep 22 23:22:14.917 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.917 INFO [0] client ExtentRepair { repair_id: ReconciliationId(42), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.917 INFO [0] Sending repair request ReconciliationId(42)
Sep 22 23:22:14.917 INFO [1] received reconcile message
Sep 22 23:22:14.917 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.917 INFO [1] client ExtentRepair { repair_id: ReconciliationId(42), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.917 INFO [1] No action required ReconciliationId(42)
Sep 22 23:22:14.917 INFO [2] received reconcile message
Sep 22 23:22:14.917 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(42), op: ExtentRepair { repair_id: ReconciliationId(42), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.917 INFO [2] client ExtentRepair { repair_id: ReconciliationId(42), extent_id: 147, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.917 INFO [2] No action required ReconciliationId(42)
Sep 22 23:22:14.917 DEBG 42 Repair extent 147 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.917 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/093.copy"
Sep 22 23:22:14.982 INFO accepted connection, remote_addr: 127.0.0.1:59665, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.982 TRCE incoming request, uri: /extent/147/files, method: GET, req_id: 53ef9fd5-b327-4d2b-ac4f-2f06221d81cc, remote_addr: 127.0.0.1:59665, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.982 INFO request completed, latency_us: 200, response_code: 200, uri: /extent/147/files, method: GET, req_id: 53ef9fd5-b327-4d2b-ac4f-2f06221d81cc, remote_addr: 127.0.0.1:59665, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.982 INFO eid:147 Found repair files: ["093", "093.db"]
Sep 22 23:22:14.983 TRCE incoming request, uri: /newextent/147/data, method: GET, req_id: 893f21d5-41de-41d6-8990-099df3e5d8c3, remote_addr: 127.0.0.1:59665, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.983 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/147/data, method: GET, req_id: 893f21d5-41de-41d6-8990-099df3e5d8c3, remote_addr: 127.0.0.1:59665, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.988 TRCE incoming request, uri: /newextent/147/db, method: GET, req_id: 1a9d2449-9c49-4b0c-ac50-95e955c41522, remote_addr: 127.0.0.1:59665, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.988 INFO request completed, latency_us: 286, response_code: 200, uri: /newextent/147/db, method: GET, req_id: 1a9d2449-9c49-4b0c-ac50-95e955c41522, remote_addr: 127.0.0.1:59665, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:14.990 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/093.copy" to "/tmp/downstairs-zrMnlo6G/00/000/093.replace"
Sep 22 23:22:14.990 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.990 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/093.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.991 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/093"
Sep 22 23:22:14.991 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/093.db"
Sep 22 23:22:14.991 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.991 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/093.replace" to "/tmp/downstairs-zrMnlo6G/00/000/093.completed"
Sep 22 23:22:14.991 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.991 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:14.991 DEBG [0] It's time to notify for 42
Sep 22 23:22:14.991 INFO Completion from [0] id:42 status:true
Sep 22 23:22:14.991 INFO [43/752] Repair commands completed
Sep 22 23:22:14.991 INFO Pop front: ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 147 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.991 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.991 INFO [0] received reconcile message
Sep 22 23:22:14.991 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 147 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.991 INFO [0] client ExtentReopen { repair_id: ReconciliationId(43), extent_id: 147 }
Sep 22 23:22:14.991 INFO [1] received reconcile message
Sep 22 23:22:14.991 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 147 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.991 INFO [1] client ExtentReopen { repair_id: ReconciliationId(43), extent_id: 147 }
Sep 22 23:22:14.991 INFO [2] received reconcile message
Sep 22 23:22:14.991 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(43), op: ExtentReopen { repair_id: ReconciliationId(43), extent_id: 147 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.991 INFO [2] client ExtentReopen { repair_id: ReconciliationId(43), extent_id: 147 }
Sep 22 23:22:14.992 DEBG 43 Reopen extent 147
Sep 22 23:22:14.992 DEBG 43 Reopen extent 147
Sep 22 23:22:14.993 DEBG 43 Reopen extent 147
Sep 22 23:22:14.993 DEBG [2] It's time to notify for 43
Sep 22 23:22:14.993 INFO Completion from [2] id:43 status:true
Sep 22 23:22:14.993 INFO [44/752] Repair commands completed
Sep 22 23:22:14.993 INFO Pop front: ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.993 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.993 INFO [0] received reconcile message
Sep 22 23:22:14.993 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.993 INFO [0] client ExtentFlush { repair_id: ReconciliationId(44), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.993 INFO [1] received reconcile message
Sep 22 23:22:14.993 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:14.993 INFO [1] client ExtentFlush { repair_id: ReconciliationId(44), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.993 INFO [2] received reconcile message
Sep 22 23:22:14.993 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(44), op: ExtentFlush { repair_id: ReconciliationId(44), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.993 INFO [2] client ExtentFlush { repair_id: ReconciliationId(44), extent_id: 67, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:14.994 DEBG 44 Flush extent 67 with f:2 g:2
Sep 22 23:22:14.994 DEBG Flush just extent 67 with f:2 and g:2
Sep 22 23:22:14.994 DEBG [1] It's time to notify for 44
Sep 22 23:22:14.994 INFO Completion from [1] id:44 status:true
Sep 22 23:22:14.994 INFO [45/752] Repair commands completed
Sep 22 23:22:14.994 INFO Pop front: ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 67 }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.994 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.994 INFO [0] received reconcile message
Sep 22 23:22:14.994 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 67 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.994 INFO [0] client ExtentClose { repair_id: ReconciliationId(45), extent_id: 67 }
Sep 22 23:22:14.994 INFO [1] received reconcile message
Sep 22 23:22:14.994 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 67 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.994 INFO [1] client ExtentClose { repair_id: ReconciliationId(45), extent_id: 67 }
Sep 22 23:22:14.994 INFO [2] received reconcile message
Sep 22 23:22:14.994 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(45), op: ExtentClose { repair_id: ReconciliationId(45), extent_id: 67 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:14.994 INFO [2] client ExtentClose { repair_id: ReconciliationId(45), extent_id: 67 }
Sep 22 23:22:14.994 DEBG 45 Close extent 67
Sep 22 23:22:14.994 DEBG 45 Close extent 67
Sep 22 23:22:14.995 DEBG 45 Close extent 67
Sep 22 23:22:14.995 DEBG [2] It's time to notify for 45
Sep 22 23:22:14.995 INFO Completion from [2] id:45 status:true
Sep 22 23:22:14.995 INFO [46/752] Repair commands completed
Sep 22 23:22:14.995 INFO Pop front: ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:14.995 INFO Sent repair work, now wait for resp
Sep 22 23:22:14.995 INFO [0] received reconcile message
Sep 22 23:22:14.995 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:14.995 INFO [0] client ExtentRepair { repair_id: ReconciliationId(46), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.995 INFO [0] Sending repair request ReconciliationId(46)
Sep 22 23:22:14.995 INFO [1] received reconcile message
Sep 22 23:22:14.995 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:14.995 INFO [1] client ExtentRepair { repair_id: ReconciliationId(46), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.995 INFO [1] No action required ReconciliationId(46)
Sep 22 23:22:14.995 INFO [2] received reconcile message
Sep 22 23:22:14.995 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(46), op: ExtentRepair { repair_id: ReconciliationId(46), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:14.995 INFO [2] client ExtentRepair { repair_id: ReconciliationId(46), extent_id: 67, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:14.995 INFO [2] No action required ReconciliationId(46)
Sep 22 23:22:14.996 DEBG 46 Repair extent 67 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:14.996 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/043.copy"
Sep 22 23:22:15.060 INFO accepted connection, remote_addr: 127.0.0.1:33675, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.060 TRCE incoming request, uri: /extent/67/files, method: GET, req_id: 3d0c8210-afa1-48c4-8e18-c4a6b56d50c5, remote_addr: 127.0.0.1:33675, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.060 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 looper connected, looper: 0
Sep 22 23:22:15.060 INFO [0] Proc runs for 127.0.0.1:42129 in state Replaced
Sep 22 23:22:15.060 INFO accepted connection from 127.0.0.1:33363, task: main
Sep 22 23:22:15.060 INFO request completed, latency_us: 400, response_code: 200, uri: /extent/67/files, method: GET, req_id: 3d0c8210-afa1-48c4-8e18-c4a6b56d50c5, remote_addr: 127.0.0.1:33675, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.061 INFO eid:67 Found repair files: ["043", "043.db"]
Sep 22 23:22:15.061 TRCE incoming request, uri: /newextent/67/data, method: GET, req_id: b8bfbe25-c2cb-4746-917f-dac2a72e966c, remote_addr: 127.0.0.1:33675, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.061 INFO request completed, latency_us: 340, response_code: 200, uri: /newextent/67/data, method: GET, req_id: b8bfbe25-c2cb-4746-917f-dac2a72e966c, remote_addr: 127.0.0.1:33675, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.067 TRCE incoming request, uri: /newextent/67/db, method: GET, req_id: 2849dfc1-62ce-4081-b43e-dd85371c722b, remote_addr: 127.0.0.1:33675, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.067 INFO request completed, latency_us: 389, response_code: 200, uri: /newextent/67/db, method: GET, req_id: 2849dfc1-62ce-4081-b43e-dd85371c722b, remote_addr: 127.0.0.1:33675, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.068 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/043.copy" to "/tmp/downstairs-zrMnlo6G/00/000/043.replace"
Sep 22 23:22:15.068 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.069 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/043.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.069 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/043"
Sep 22 23:22:15.069 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/043.db"
Sep 22 23:22:15.069 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.069 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/043.replace" to "/tmp/downstairs-zrMnlo6G/00/000/043.completed"
Sep 22 23:22:15.069 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.069 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.069 INFO Connection request from 9144e02c-c312-47c4-9b1c-f03618834608 with version 4, task: proc
Sep 22 23:22:15.070 INFO upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } connected, version 4, task: proc
Sep 22 23:22:15.070 DEBG [0] It's time to notify for 46
Sep 22 23:22:15.070 INFO [0] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) Replaced Disconnected Disconnected ds_transition to WaitActive
thread 'test::integration_test_volume_replace_downstairs_then_takeover' panicked at '[0] 9144e02c-c312-47c4-9b1c-f03618834608 Initializing Negotiation failed, Replaced -> WaitActive', upstairs/src/lib.rs:6349:21
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
Sep 22 23:22:15.070 INFO Completion from [0] id:46 status:true
Sep 22 23:22:15.070 INFO [47/752] Repair commands completed
Sep 22 23:22:15.070 INFO Pop front: ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 67 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.070 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.070 INFO [0] received reconcile message
Sep 22 23:22:15.070 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 67 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.070 INFO [0] client ExtentReopen { repair_id: ReconciliationId(47), extent_id: 67 }
Sep 22 23:22:15.070 INFO [1] received reconcile message
Sep 22 23:22:15.071 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 67 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.071 INFO [1] client ExtentReopen { repair_id: ReconciliationId(47), extent_id: 67 }
Sep 22 23:22:15.071 INFO [2] received reconcile message
Sep 22 23:22:15.071 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(47), op: ExtentReopen { repair_id: ReconciliationId(47), extent_id: 67 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.071 INFO [2] client ExtentReopen { repair_id: ReconciliationId(47), extent_id: 67 }
Sep 22 23:22:15.071 WARN UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } cannot grab lock, does not match UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: ec8dc28c-8140-4e2d-b538-f84f8abf147b, gen: 2 }!
Sep 22 23:22:15.071 INFO upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } disconnected, Upstairs is not active, task: proc
Sep 22 23:22:15.071 INFO connection (127.0.0.1:33363): all done
Sep 22 23:22:15.071 DEBG 47 Reopen extent 67
Sep 22 23:22:15.072 DEBG 47 Reopen extent 67
Sep 22 23:22:15.072 DEBG 47 Reopen extent 67
Sep 22 23:22:15.073 DEBG [2] It's time to notify for 47
Sep 22 23:22:15.073 INFO Completion from [2] id:47 status:true
Sep 22 23:22:15.073 INFO [48/752] Repair commands completed
Sep 22 23:22:15.073 INFO Pop front: ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.073 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.073 INFO [0] received reconcile message
Sep 22 23:22:15.073 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.073 INFO [0] client ExtentFlush { repair_id: ReconciliationId(48), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.073 INFO [1] received reconcile message
Sep 22 23:22:15.073 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.073 INFO [1] client ExtentFlush { repair_id: ReconciliationId(48), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.073 INFO [2] received reconcile message
Sep 22 23:22:15.073 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(48), op: ExtentFlush { repair_id: ReconciliationId(48), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.073 INFO [2] client ExtentFlush { repair_id: ReconciliationId(48), extent_id: 2, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.073 DEBG 48 Flush extent 2 with f:2 g:2
Sep 22 23:22:15.073 DEBG Flush just extent 2 with f:2 and g:2
Sep 22 23:22:15.073 DEBG [1] It's time to notify for 48
Sep 22 23:22:15.073 INFO Completion from [1] id:48 status:true
Sep 22 23:22:15.073 INFO [49/752] Repair commands completed
Sep 22 23:22:15.073 INFO Pop front: ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.073 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.073 INFO [0] received reconcile message
Sep 22 23:22:15.073 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.073 INFO [0] client ExtentClose { repair_id: ReconciliationId(49), extent_id: 2 }
Sep 22 23:22:15.073 INFO [1] received reconcile message
Sep 22 23:22:15.073 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 2 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.074 INFO [1] client ExtentClose { repair_id: ReconciliationId(49), extent_id: 2 }
Sep 22 23:22:15.074 INFO [2] received reconcile message
Sep 22 23:22:15.074 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(49), op: ExtentClose { repair_id: ReconciliationId(49), extent_id: 2 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.074 INFO [2] client ExtentClose { repair_id: ReconciliationId(49), extent_id: 2 }
Sep 22 23:22:15.074 DEBG 49 Close extent 2
Sep 22 23:22:15.074 DEBG 49 Close extent 2
Sep 22 23:22:15.074 DEBG 49 Close extent 2
Sep 22 23:22:15.075 DEBG [2] It's time to notify for 49
Sep 22 23:22:15.075 INFO Completion from [2] id:49 status:true
Sep 22 23:22:15.075 INFO [50/752] Repair commands completed
Sep 22 23:22:15.075 INFO Pop front: ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.075 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.075 INFO [0] received reconcile message
Sep 22 23:22:15.075 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.075 INFO [0] client ExtentRepair { repair_id: ReconciliationId(50), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.075 INFO [0] Sending repair request ReconciliationId(50)
Sep 22 23:22:15.075 INFO [1] received reconcile message
Sep 22 23:22:15.075 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.075 INFO [1] client ExtentRepair { repair_id: ReconciliationId(50), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.075 INFO [1] No action required ReconciliationId(50)
Sep 22 23:22:15.075 INFO [2] received reconcile message
Sep 22 23:22:15.075 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(50), op: ExtentRepair { repair_id: ReconciliationId(50), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.075 INFO [2] client ExtentRepair { repair_id: ReconciliationId(50), extent_id: 2, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.075 INFO [2] No action required ReconciliationId(50)
Sep 22 23:22:15.075 DEBG 50 Repair extent 2 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.075 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/002.copy"
Sep 22 23:22:15.117 DEBG [0] Read AckReady 1073, : downstairs
Sep 22 23:22:15.118 DEBG up_ds_listen was notified
Sep 22 23:22:15.118 DEBG up_ds_listen process 1073
Sep 22 23:22:15.118 DEBG [A] ack job 1073:74, : downstairs
Sep 22 23:22:15.137 INFO accepted connection, remote_addr: 127.0.0.1:44318, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.137 TRCE incoming request, uri: /extent/2/files, method: GET, req_id: ce42ef6e-a041-4c31-bc0c-64c8f3ce57b2, remote_addr: 127.0.0.1:44318, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.137 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 looper connected, looper: 1
Sep 22 23:22:15.138 INFO [1] Proc runs for 127.0.0.1:48339 in state Disconnected
Sep 22 23:22:15.138 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 looper connected, looper: 2
Sep 22 23:22:15.138 INFO [2] Proc runs for 127.0.0.1:33021 in state Disconnected
Sep 22 23:22:15.138 INFO accepted connection from 127.0.0.1:42838, task: main
Sep 22 23:22:15.138 INFO accepted connection from 127.0.0.1:44046, task: main
Sep 22 23:22:15.138 INFO request completed, latency_us: 521, response_code: 200, uri: /extent/2/files, method: GET, req_id: ce42ef6e-a041-4c31-bc0c-64c8f3ce57b2, remote_addr: 127.0.0.1:44318, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.138 INFO Connection request from 9144e02c-c312-47c4-9b1c-f03618834608 with version 4, task: proc
Sep 22 23:22:15.138 INFO upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } connected, version 4, task: proc
Sep 22 23:22:15.138 INFO Connection request from 9144e02c-c312-47c4-9b1c-f03618834608 with version 4, task: proc
Sep 22 23:22:15.138 INFO upstairs UpstairsConnection { upstairs_id: 9144e02c-c312-47c4-9b1c-f03618834608, session_id: cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f, gen: 1 } connected, version 4, task: proc
Sep 22 23:22:15.138 INFO eid:2 Found repair files: ["002", "002.db"]
Sep 22 23:22:15.139 TRCE incoming request, uri: /newextent/2/data, method: GET, req_id: 3308ce35-403c-4e61-9152-eeec5a770056, remote_addr: 127.0.0.1:44318, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.139 INFO [1] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) Replaced Disconnected Disconnected ds_transition to WaitActive
Sep 22 23:22:15.139 INFO [1] Transition from Disconnected to WaitActive
Sep 22 23:22:15.139 INFO [2] 9144e02c-c312-47c4-9b1c-f03618834608 (cdb4ad0d-846c-40eb-b2e0-a9a03f8a5f6f) Replaced WaitActive Disconnected ds_transition to WaitActive
Sep 22 23:22:15.139 INFO [2] Transition from Disconnected to WaitActive
Sep 22 23:22:15.139 INFO request completed, latency_us: 481, response_code: 200, uri: /newextent/2/data, method: GET, req_id: 3308ce35-403c-4e61-9152-eeec5a770056, remote_addr: 127.0.0.1:44318, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.144 TRCE incoming request, uri: /newextent/2/db, method: GET, req_id: 0bac4694-f818-4f86-895d-2c5a3b34d737, remote_addr: 127.0.0.1:44318, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.144 INFO request completed, latency_us: 287, response_code: 200, uri: /newextent/2/db, method: GET, req_id: 0bac4694-f818-4f86-895d-2c5a3b34d737, remote_addr: 127.0.0.1:44318, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.146 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/002.copy" to "/tmp/downstairs-zrMnlo6G/00/000/002.replace"
Sep 22 23:22:15.146 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.146 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/002.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.147 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/002"
Sep 22 23:22:15.147 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/002.db"
Sep 22 23:22:15.147 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.147 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/002.replace" to "/tmp/downstairs-zrMnlo6G/00/000/002.completed"
Sep 22 23:22:15.147 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.147 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.147 DEBG [0] It's time to notify for 50
Sep 22 23:22:15.147 INFO Completion from [0] id:50 status:true
Sep 22 23:22:15.147 INFO [51/752] Repair commands completed
Sep 22 23:22:15.147 INFO Pop front: ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.147 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.147 INFO [0] received reconcile message
Sep 22 23:22:15.147 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.147 INFO [0] client ExtentReopen { repair_id: ReconciliationId(51), extent_id: 2 }
Sep 22 23:22:15.147 INFO [1] received reconcile message
Sep 22 23:22:15.147 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 2 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.147 INFO [1] client ExtentReopen { repair_id: ReconciliationId(51), extent_id: 2 }
Sep 22 23:22:15.147 INFO [2] received reconcile message
Sep 22 23:22:15.147 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(51), op: ExtentReopen { repair_id: ReconciliationId(51), extent_id: 2 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.147 INFO [2] client ExtentReopen { repair_id: ReconciliationId(51), extent_id: 2 }
Sep 22 23:22:15.148 DEBG 51 Reopen extent 2
Sep 22 23:22:15.148 DEBG 51 Reopen extent 2
Sep 22 23:22:15.149 DEBG 51 Reopen extent 2
Sep 22 23:22:15.149 DEBG [2] It's time to notify for 51
Sep 22 23:22:15.149 INFO Completion from [2] id:51 status:true
Sep 22 23:22:15.149 INFO [52/752] Repair commands completed
Sep 22 23:22:15.149 INFO Pop front: ReconcileIO { id: ReconciliationId(52), op: ExtentFlush { repair_id: ReconciliationId(52), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.149 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.149 INFO [0] received reconcile message
Sep 22 23:22:15.149 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(52), op: ExtentFlush { repair_id: ReconciliationId(52), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.149 INFO [0] client ExtentFlush { repair_id: ReconciliationId(52), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.149 INFO [1] received reconcile message
Sep 22 23:22:15.149 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(52), op: ExtentFlush { repair_id: ReconciliationId(52), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.149 INFO [1] client ExtentFlush { repair_id: ReconciliationId(52), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.149 INFO [2] received reconcile message
Sep 22 23:22:15.149 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(52), op: ExtentFlush { repair_id: ReconciliationId(52), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.149 INFO [2] client ExtentFlush { repair_id: ReconciliationId(52), extent_id: 25, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.150 DEBG 52 Flush extent 25 with f:2 g:2
Sep 22 23:22:15.150 DEBG Flush just extent 25 with f:2 and g:2
Sep 22 23:22:15.150 DEBG [1] It's time to notify for 52
Sep 22 23:22:15.150 INFO Completion from [1] id:52 status:true
Sep 22 23:22:15.150 INFO [53/752] Repair commands completed
Sep 22 23:22:15.150 INFO Pop front: ReconcileIO { id: ReconciliationId(53), op: ExtentClose { repair_id: ReconciliationId(53), extent_id: 25 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.150 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.150 INFO [0] received reconcile message
Sep 22 23:22:15.150 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(53), op: ExtentClose { repair_id: ReconciliationId(53), extent_id: 25 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.150 INFO [0] client ExtentClose { repair_id: ReconciliationId(53), extent_id: 25 }
Sep 22 23:22:15.150 INFO [1] received reconcile message
Sep 22 23:22:15.150 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(53), op: ExtentClose { repair_id: ReconciliationId(53), extent_id: 25 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.150 INFO [1] client ExtentClose { repair_id: ReconciliationId(53), extent_id: 25 }
Sep 22 23:22:15.150 INFO [2] received reconcile message
Sep 22 23:22:15.150 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(53), op: ExtentClose { repair_id: ReconciliationId(53), extent_id: 25 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.150 INFO [2] client ExtentClose { repair_id: ReconciliationId(53), extent_id: 25 }
Sep 22 23:22:15.150 DEBG 53 Close extent 25
Sep 22 23:22:15.150 DEBG 53 Close extent 25
Sep 22 23:22:15.151 DEBG 53 Close extent 25
Sep 22 23:22:15.151 DEBG [2] It's time to notify for 53
Sep 22 23:22:15.151 INFO Completion from [2] id:53 status:true
Sep 22 23:22:15.151 INFO [54/752] Repair commands completed
Sep 22 23:22:15.151 INFO Pop front: ReconcileIO { id: ReconciliationId(54), op: ExtentRepair { repair_id: ReconciliationId(54), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.151 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.151 INFO [0] received reconcile message
Sep 22 23:22:15.151 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(54), op: ExtentRepair { repair_id: ReconciliationId(54), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.151 INFO [0] client ExtentRepair { repair_id: ReconciliationId(54), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.151 INFO [0] Sending repair request ReconciliationId(54)
Sep 22 23:22:15.151 INFO [1] received reconcile message
Sep 22 23:22:15.151 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(54), op: ExtentRepair { repair_id: ReconciliationId(54), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.151 INFO [1] client ExtentRepair { repair_id: ReconciliationId(54), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.151 INFO [1] No action required ReconciliationId(54)
Sep 22 23:22:15.151 INFO [2] received reconcile message
Sep 22 23:22:15.151 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(54), op: ExtentRepair { repair_id: ReconciliationId(54), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.151 INFO [2] client ExtentRepair { repair_id: ReconciliationId(54), extent_id: 25, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.151 INFO [2] No action required ReconciliationId(54)
Sep 22 23:22:15.152 DEBG 54 Repair extent 25 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.152 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/019.copy"
Sep 22 23:22:15.171 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:15.178 DEBG Read      :1073 deps:[JobId(1072)] res:true
Sep 22 23:22:15.199 DEBG Flush     :1072 extent_limit None deps:[JobId(1071), JobId(1070)] res:true f:25 g:1
Sep 22 23:22:15.199 INFO [lossy] skipping 1073
Sep 22 23:22:15.199 INFO [lossy] skipping 1073
Sep 22 23:22:15.206 DEBG Read      :1073 deps:[JobId(1072)] res:true
Sep 22 23:22:15.215 INFO accepted connection, remote_addr: 127.0.0.1:61012, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.215 TRCE incoming request, uri: /extent/25/files, method: GET, req_id: 9e179dd3-65c4-47fe-8168-ad8fdf6dde8a, remote_addr: 127.0.0.1:61012, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.215 INFO request completed, latency_us: 213, response_code: 200, uri: /extent/25/files, method: GET, req_id: 9e179dd3-65c4-47fe-8168-ad8fdf6dde8a, remote_addr: 127.0.0.1:61012, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.215 INFO eid:25 Found repair files: ["019", "019.db"]
Sep 22 23:22:15.216 TRCE incoming request, uri: /newextent/25/data, method: GET, req_id: 00fefafb-175a-442e-9964-f817695700bb, remote_addr: 127.0.0.1:61012, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.216 INFO request completed, latency_us: 320, response_code: 200, uri: /newextent/25/data, method: GET, req_id: 00fefafb-175a-442e-9964-f817695700bb, remote_addr: 127.0.0.1:61012, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.221 TRCE incoming request, uri: /newextent/25/db, method: GET, req_id: f35c4604-c36f-4059-8ac9-848a04cc458a, remote_addr: 127.0.0.1:61012, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.221 INFO request completed, latency_us: 289, response_code: 200, uri: /newextent/25/db, method: GET, req_id: f35c4604-c36f-4059-8ac9-848a04cc458a, remote_addr: 127.0.0.1:61012, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.222 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/019.copy" to "/tmp/downstairs-zrMnlo6G/00/000/019.replace"
Sep 22 23:22:15.222 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.223 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/019.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.223 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/019"
Sep 22 23:22:15.224 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/019.db"
Sep 22 23:22:15.224 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.224 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/019.replace" to "/tmp/downstairs-zrMnlo6G/00/000/019.completed"
Sep 22 23:22:15.224 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.224 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.224 DEBG [0] It's time to notify for 54
Sep 22 23:22:15.224 INFO Completion from [0] id:54 status:true
Sep 22 23:22:15.224 INFO [55/752] Repair commands completed
Sep 22 23:22:15.224 INFO Pop front: ReconcileIO { id: ReconciliationId(55), op: ExtentReopen { repair_id: ReconciliationId(55), extent_id: 25 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.224 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.224 INFO [0] received reconcile message
Sep 22 23:22:15.224 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(55), op: ExtentReopen { repair_id: ReconciliationId(55), extent_id: 25 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.224 INFO [0] client ExtentReopen { repair_id: ReconciliationId(55), extent_id: 25 }
Sep 22 23:22:15.224 INFO [1] received reconcile message
Sep 22 23:22:15.224 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(55), op: ExtentReopen { repair_id: ReconciliationId(55), extent_id: 25 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.224 INFO [1] client ExtentReopen { repair_id: ReconciliationId(55), extent_id: 25 }
Sep 22 23:22:15.224 INFO [2] received reconcile message
Sep 22 23:22:15.224 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(55), op: ExtentReopen { repair_id: ReconciliationId(55), extent_id: 25 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.224 INFO [2] client ExtentReopen { repair_id: ReconciliationId(55), extent_id: 25 }
Sep 22 23:22:15.224 DEBG 55 Reopen extent 25
Sep 22 23:22:15.225 DEBG 55 Reopen extent 25
Sep 22 23:22:15.226 DEBG 55 Reopen extent 25
Sep 22 23:22:15.226 DEBG [2] It's time to notify for 55
Sep 22 23:22:15.226 INFO Completion from [2] id:55 status:true
Sep 22 23:22:15.226 INFO [56/752] Repair commands completed
Sep 22 23:22:15.226 INFO Pop front: ReconcileIO { id: ReconciliationId(56), op: ExtentFlush { repair_id: ReconciliationId(56), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.226 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.226 INFO [0] received reconcile message
Sep 22 23:22:15.226 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(56), op: ExtentFlush { repair_id: ReconciliationId(56), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.226 INFO [0] client ExtentFlush { repair_id: ReconciliationId(56), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.226 INFO [1] received reconcile message
Sep 22 23:22:15.226 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(56), op: ExtentFlush { repair_id: ReconciliationId(56), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.226 INFO [1] client ExtentFlush { repair_id: ReconciliationId(56), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.227 INFO [2] received reconcile message
Sep 22 23:22:15.227 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(56), op: ExtentFlush { repair_id: ReconciliationId(56), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.227 INFO [2] client ExtentFlush { repair_id: ReconciliationId(56), extent_id: 152, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.227 DEBG 56 Flush extent 152 with f:2 g:2
Sep 22 23:22:15.227 DEBG Flush just extent 152 with f:2 and g:2
Sep 22 23:22:15.227 DEBG [1] It's time to notify for 56
Sep 22 23:22:15.227 INFO Completion from [1] id:56 status:true
Sep 22 23:22:15.227 INFO [57/752] Repair commands completed
Sep 22 23:22:15.227 INFO Pop front: ReconcileIO { id: ReconciliationId(57), op: ExtentClose { repair_id: ReconciliationId(57), extent_id: 152 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.227 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.227 INFO [0] received reconcile message
Sep 22 23:22:15.227 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(57), op: ExtentClose { repair_id: ReconciliationId(57), extent_id: 152 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.227 INFO [0] client ExtentClose { repair_id: ReconciliationId(57), extent_id: 152 }
Sep 22 23:22:15.227 INFO [1] received reconcile message
Sep 22 23:22:15.227 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(57), op: ExtentClose { repair_id: ReconciliationId(57), extent_id: 152 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.227 INFO [1] client ExtentClose { repair_id: ReconciliationId(57), extent_id: 152 }
Sep 22 23:22:15.227 INFO [2] received reconcile message
Sep 22 23:22:15.227 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(57), op: ExtentClose { repair_id: ReconciliationId(57), extent_id: 152 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.227 INFO [2] client ExtentClose { repair_id: ReconciliationId(57), extent_id: 152 }
Sep 22 23:22:15.227 DEBG 57 Close extent 152
Sep 22 23:22:15.227 DEBG IO Read  1075 has deps [JobId(1074)]
Sep 22 23:22:15.227 DEBG IO Flush 1076 has deps [JobId(1075), JobId(1074)]
Sep 22 23:22:15.227 DEBG 57 Close extent 152
Sep 22 23:22:15.228 DEBG [rc] retire 1072 clears [JobId(1071), JobId(1072)], : downstairs
Sep 22 23:22:15.228 DEBG 57 Close extent 152
Sep 22 23:22:15.228 DEBG [2] It's time to notify for 57
Sep 22 23:22:15.228 INFO Completion from [2] id:57 status:true
Sep 22 23:22:15.228 INFO [58/752] Repair commands completed
Sep 22 23:22:15.228 INFO Pop front: ReconcileIO { id: ReconciliationId(58), op: ExtentRepair { repair_id: ReconciliationId(58), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.228 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.228 INFO [0] received reconcile message
Sep 22 23:22:15.228 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(58), op: ExtentRepair { repair_id: ReconciliationId(58), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.228 INFO [0] client ExtentRepair { repair_id: ReconciliationId(58), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.228 INFO [0] Sending repair request ReconciliationId(58)
Sep 22 23:22:15.228 INFO [1] received reconcile message
Sep 22 23:22:15.228 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(58), op: ExtentRepair { repair_id: ReconciliationId(58), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.228 INFO [1] client ExtentRepair { repair_id: ReconciliationId(58), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.228 INFO [1] No action required ReconciliationId(58)
Sep 22 23:22:15.229 INFO [2] received reconcile message
Sep 22 23:22:15.229 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(58), op: ExtentRepair { repair_id: ReconciliationId(58), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.229 INFO [2] client ExtentRepair { repair_id: ReconciliationId(58), extent_id: 152, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.229 INFO [2] No action required ReconciliationId(58)
Sep 22 23:22:15.229 DEBG 58 Repair extent 152 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.229 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/098.copy"
Sep 22 23:22:15.242 WARN returning error on flush!
Sep 22 23:22:15.242 DEBG Flush     :1074 extent_limit None deps:[JobId(1073), JobId(1072)] res:false f:26 g:1
Sep 22 23:22:15.242 INFO [lossy] skipping 1074
Sep 22 23:22:15.242 DEBG Flush     :1074 extent_limit None deps:[JobId(1073), JobId(1072)] res:true f:26 g:1
Sep 22 23:22:15.242 INFO [lossy] skipping 1075
Sep 22 23:22:15.248 DEBG Read      :1075 deps:[JobId(1074)] res:true
Sep 22 23:22:15.270 INFO [lossy] sleeping 1 second
Sep 22 23:22:15.290 INFO accepted connection, remote_addr: 127.0.0.1:36859, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.290 TRCE incoming request, uri: /extent/152/files, method: GET, req_id: e23c5b15-f803-483f-a1fb-369149642967, remote_addr: 127.0.0.1:36859, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.291 INFO request completed, latency_us: 271, response_code: 200, uri: /extent/152/files, method: GET, req_id: e23c5b15-f803-483f-a1fb-369149642967, remote_addr: 127.0.0.1:36859, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.291 INFO eid:152 Found repair files: ["098", "098.db"]
Sep 22 23:22:15.291 TRCE incoming request, uri: /newextent/152/data, method: GET, req_id: 3812f098-8d70-4b13-8b92-a01bf068aa03, remote_addr: 127.0.0.1:36859, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.292 INFO request completed, latency_us: 359, response_code: 200, uri: /newextent/152/data, method: GET, req_id: 3812f098-8d70-4b13-8b92-a01bf068aa03, remote_addr: 127.0.0.1:36859, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.297 TRCE incoming request, uri: /newextent/152/db, method: GET, req_id: 09656b5a-b661-4cc9-8a44-78a034cb4528, remote_addr: 127.0.0.1:36859, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.297 INFO request completed, latency_us: 287, response_code: 200, uri: /newextent/152/db, method: GET, req_id: 09656b5a-b661-4cc9-8a44-78a034cb4528, remote_addr: 127.0.0.1:36859, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.298 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/098.copy" to "/tmp/downstairs-zrMnlo6G/00/000/098.replace"
Sep 22 23:22:15.298 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.299 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/098.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.299 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/098"
Sep 22 23:22:15.300 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/098.db"
Sep 22 23:22:15.300 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.300 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/098.replace" to "/tmp/downstairs-zrMnlo6G/00/000/098.completed"
Sep 22 23:22:15.300 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.300 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.300 DEBG [0] It's time to notify for 58
Sep 22 23:22:15.300 INFO Completion from [0] id:58 status:true
Sep 22 23:22:15.300 INFO [59/752] Repair commands completed
Sep 22 23:22:15.300 INFO Pop front: ReconcileIO { id: ReconciliationId(59), op: ExtentReopen { repair_id: ReconciliationId(59), extent_id: 152 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.300 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.300 INFO [0] received reconcile message
Sep 22 23:22:15.300 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(59), op: ExtentReopen { repair_id: ReconciliationId(59), extent_id: 152 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.300 INFO [0] client ExtentReopen { repair_id: ReconciliationId(59), extent_id: 152 }
Sep 22 23:22:15.300 INFO [1] received reconcile message
Sep 22 23:22:15.300 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(59), op: ExtentReopen { repair_id: ReconciliationId(59), extent_id: 152 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.300 INFO [1] client ExtentReopen { repair_id: ReconciliationId(59), extent_id: 152 }
Sep 22 23:22:15.300 INFO [2] received reconcile message
Sep 22 23:22:15.300 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(59), op: ExtentReopen { repair_id: ReconciliationId(59), extent_id: 152 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.300 INFO [2] client ExtentReopen { repair_id: ReconciliationId(59), extent_id: 152 }
Sep 22 23:22:15.301 DEBG 59 Reopen extent 152
Sep 22 23:22:15.301 DEBG 59 Reopen extent 152
Sep 22 23:22:15.302 DEBG 59 Reopen extent 152
Sep 22 23:22:15.302 DEBG [2] It's time to notify for 59
Sep 22 23:22:15.302 INFO Completion from [2] id:59 status:true
Sep 22 23:22:15.302 INFO [60/752] Repair commands completed
Sep 22 23:22:15.302 INFO Pop front: ReconcileIO { id: ReconciliationId(60), op: ExtentFlush { repair_id: ReconciliationId(60), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.303 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.303 INFO [0] received reconcile message
Sep 22 23:22:15.303 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(60), op: ExtentFlush { repair_id: ReconciliationId(60), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.303 INFO [0] client ExtentFlush { repair_id: ReconciliationId(60), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.303 INFO [1] received reconcile message
Sep 22 23:22:15.303 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(60), op: ExtentFlush { repair_id: ReconciliationId(60), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.303 INFO [1] client ExtentFlush { repair_id: ReconciliationId(60), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.303 INFO [2] received reconcile message
Sep 22 23:22:15.303 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(60), op: ExtentFlush { repair_id: ReconciliationId(60), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.303 INFO [2] client ExtentFlush { repair_id: ReconciliationId(60), extent_id: 153, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.303 DEBG 60 Flush extent 153 with f:2 g:2
Sep 22 23:22:15.303 DEBG Flush just extent 153 with f:2 and g:2
Sep 22 23:22:15.303 DEBG [1] It's time to notify for 60
Sep 22 23:22:15.303 INFO Completion from [1] id:60 status:true
Sep 22 23:22:15.303 INFO [61/752] Repair commands completed
Sep 22 23:22:15.303 INFO Pop front: ReconcileIO { id: ReconciliationId(61), op: ExtentClose { repair_id: ReconciliationId(61), extent_id: 153 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.303 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.303 INFO [0] received reconcile message
Sep 22 23:22:15.303 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(61), op: ExtentClose { repair_id: ReconciliationId(61), extent_id: 153 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.303 INFO [0] client ExtentClose { repair_id: ReconciliationId(61), extent_id: 153 }
Sep 22 23:22:15.303 INFO [1] received reconcile message
Sep 22 23:22:15.303 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(61), op: ExtentClose { repair_id: ReconciliationId(61), extent_id: 153 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.303 INFO [1] client ExtentClose { repair_id: ReconciliationId(61), extent_id: 153 }
Sep 22 23:22:15.303 INFO [2] received reconcile message
Sep 22 23:22:15.303 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(61), op: ExtentClose { repair_id: ReconciliationId(61), extent_id: 153 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.303 INFO [2] client ExtentClose { repair_id: ReconciliationId(61), extent_id: 153 }
Sep 22 23:22:15.303 DEBG 61 Close extent 153
Sep 22 23:22:15.304 DEBG 61 Close extent 153
Sep 22 23:22:15.304 DEBG 61 Close extent 153
Sep 22 23:22:15.304 DEBG [2] It's time to notify for 61
Sep 22 23:22:15.304 INFO Completion from [2] id:61 status:true
Sep 22 23:22:15.304 INFO [62/752] Repair commands completed
Sep 22 23:22:15.304 INFO Pop front: ReconcileIO { id: ReconciliationId(62), op: ExtentRepair { repair_id: ReconciliationId(62), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.304 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.305 INFO [0] received reconcile message
Sep 22 23:22:15.305 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(62), op: ExtentRepair { repair_id: ReconciliationId(62), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.305 INFO [0] client ExtentRepair { repair_id: ReconciliationId(62), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.305 INFO [0] Sending repair request ReconciliationId(62)
Sep 22 23:22:15.305 INFO [1] received reconcile message
Sep 22 23:22:15.305 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(62), op: ExtentRepair { repair_id: ReconciliationId(62), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.305 INFO [1] client ExtentRepair { repair_id: ReconciliationId(62), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.305 INFO [1] No action required ReconciliationId(62)
Sep 22 23:22:15.305 INFO [2] received reconcile message
Sep 22 23:22:15.305 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(62), op: ExtentRepair { repair_id: ReconciliationId(62), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.305 INFO [2] client ExtentRepair { repair_id: ReconciliationId(62), extent_id: 153, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.305 INFO [2] No action required ReconciliationId(62)
Sep 22 23:22:15.305 DEBG 62 Repair extent 153 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.305 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/099.copy"
Sep 22 23:22:15.367 INFO accepted connection, remote_addr: 127.0.0.1:42918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.368 TRCE incoming request, uri: /extent/153/files, method: GET, req_id: a2534261-e377-44aa-8f56-bd13b383c871, remote_addr: 127.0.0.1:42918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.368 INFO request completed, latency_us: 258, response_code: 200, uri: /extent/153/files, method: GET, req_id: a2534261-e377-44aa-8f56-bd13b383c871, remote_addr: 127.0.0.1:42918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.368 INFO eid:153 Found repair files: ["099", "099.db"]
Sep 22 23:22:15.369 TRCE incoming request, uri: /newextent/153/data, method: GET, req_id: 351ef690-c10c-41d9-9401-c9e825acf2a5, remote_addr: 127.0.0.1:42918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.369 INFO request completed, latency_us: 355, response_code: 200, uri: /newextent/153/data, method: GET, req_id: 351ef690-c10c-41d9-9401-c9e825acf2a5, remote_addr: 127.0.0.1:42918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.374 TRCE incoming request, uri: /newextent/153/db, method: GET, req_id: 3c84bb53-b1c7-4667-9499-442227b1160b, remote_addr: 127.0.0.1:42918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.374 INFO request completed, latency_us: 315, response_code: 200, uri: /newextent/153/db, method: GET, req_id: 3c84bb53-b1c7-4667-9499-442227b1160b, remote_addr: 127.0.0.1:42918, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.376 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/099.copy" to "/tmp/downstairs-zrMnlo6G/00/000/099.replace"
Sep 22 23:22:15.376 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.377 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/099.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.377 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/099"
Sep 22 23:22:15.377 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/099.db"
Sep 22 23:22:15.377 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.377 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/099.replace" to "/tmp/downstairs-zrMnlo6G/00/000/099.completed"
Sep 22 23:22:15.377 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.377 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.378 DEBG [0] It's time to notify for 62
Sep 22 23:22:15.378 INFO Completion from [0] id:62 status:true
Sep 22 23:22:15.378 INFO [63/752] Repair commands completed
Sep 22 23:22:15.378 INFO Pop front: ReconcileIO { id: ReconciliationId(63), op: ExtentReopen { repair_id: ReconciliationId(63), extent_id: 153 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.378 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.378 INFO [0] received reconcile message
Sep 22 23:22:15.378 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(63), op: ExtentReopen { repair_id: ReconciliationId(63), extent_id: 153 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.378 INFO [0] client ExtentReopen { repair_id: ReconciliationId(63), extent_id: 153 }
Sep 22 23:22:15.378 INFO [1] received reconcile message
Sep 22 23:22:15.378 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(63), op: ExtentReopen { repair_id: ReconciliationId(63), extent_id: 153 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.378 INFO [1] client ExtentReopen { repair_id: ReconciliationId(63), extent_id: 153 }
Sep 22 23:22:15.378 INFO [2] received reconcile message
Sep 22 23:22:15.378 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(63), op: ExtentReopen { repair_id: ReconciliationId(63), extent_id: 153 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.378 INFO [2] client ExtentReopen { repair_id: ReconciliationId(63), extent_id: 153 }
Sep 22 23:22:15.378 DEBG 63 Reopen extent 153
Sep 22 23:22:15.379 DEBG 63 Reopen extent 153
Sep 22 23:22:15.379 DEBG 63 Reopen extent 153
Sep 22 23:22:15.380 DEBG [2] It's time to notify for 63
Sep 22 23:22:15.380 INFO Completion from [2] id:63 status:true
Sep 22 23:22:15.380 INFO [64/752] Repair commands completed
Sep 22 23:22:15.380 INFO Pop front: ReconcileIO { id: ReconciliationId(64), op: ExtentFlush { repair_id: ReconciliationId(64), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.380 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.380 INFO [0] received reconcile message
Sep 22 23:22:15.380 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(64), op: ExtentFlush { repair_id: ReconciliationId(64), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.380 INFO [0] client ExtentFlush { repair_id: ReconciliationId(64), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.380 INFO [1] received reconcile message
Sep 22 23:22:15.380 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(64), op: ExtentFlush { repair_id: ReconciliationId(64), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.380 INFO [1] client ExtentFlush { repair_id: ReconciliationId(64), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.380 INFO [2] received reconcile message
Sep 22 23:22:15.380 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(64), op: ExtentFlush { repair_id: ReconciliationId(64), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.380 INFO [2] client ExtentFlush { repair_id: ReconciliationId(64), extent_id: 156, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.381 DEBG 64 Flush extent 156 with f:2 g:2
Sep 22 23:22:15.381 DEBG Flush just extent 156 with f:2 and g:2
Sep 22 23:22:15.381 DEBG [1] It's time to notify for 64
Sep 22 23:22:15.381 INFO Completion from [1] id:64 status:true
Sep 22 23:22:15.381 INFO [65/752] Repair commands completed
Sep 22 23:22:15.381 INFO Pop front: ReconcileIO { id: ReconciliationId(65), op: ExtentClose { repair_id: ReconciliationId(65), extent_id: 156 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.381 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.381 INFO [0] received reconcile message
Sep 22 23:22:15.381 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(65), op: ExtentClose { repair_id: ReconciliationId(65), extent_id: 156 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.381 INFO [0] client ExtentClose { repair_id: ReconciliationId(65), extent_id: 156 }
Sep 22 23:22:15.381 INFO [1] received reconcile message
Sep 22 23:22:15.381 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(65), op: ExtentClose { repair_id: ReconciliationId(65), extent_id: 156 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.381 INFO [1] client ExtentClose { repair_id: ReconciliationId(65), extent_id: 156 }
Sep 22 23:22:15.381 INFO [2] received reconcile message
Sep 22 23:22:15.381 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(65), op: ExtentClose { repair_id: ReconciliationId(65), extent_id: 156 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.381 INFO [2] client ExtentClose { repair_id: ReconciliationId(65), extent_id: 156 }
Sep 22 23:22:15.381 DEBG 65 Close extent 156
Sep 22 23:22:15.381 DEBG 65 Close extent 156
Sep 22 23:22:15.382 DEBG 65 Close extent 156
Sep 22 23:22:15.382 DEBG [2] It's time to notify for 65
Sep 22 23:22:15.382 INFO Completion from [2] id:65 status:true
Sep 22 23:22:15.382 INFO [66/752] Repair commands completed
Sep 22 23:22:15.382 INFO Pop front: ReconcileIO { id: ReconciliationId(66), op: ExtentRepair { repair_id: ReconciliationId(66), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.382 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.382 INFO [0] received reconcile message
Sep 22 23:22:15.382 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(66), op: ExtentRepair { repair_id: ReconciliationId(66), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.382 INFO [0] client ExtentRepair { repair_id: ReconciliationId(66), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.382 INFO [0] Sending repair request ReconciliationId(66)
Sep 22 23:22:15.382 INFO [1] received reconcile message
Sep 22 23:22:15.382 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(66), op: ExtentRepair { repair_id: ReconciliationId(66), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.383 INFO [1] client ExtentRepair { repair_id: ReconciliationId(66), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.383 INFO [1] No action required ReconciliationId(66)
Sep 22 23:22:15.383 INFO [2] received reconcile message
Sep 22 23:22:15.383 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(66), op: ExtentRepair { repair_id: ReconciliationId(66), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.383 INFO [2] client ExtentRepair { repair_id: ReconciliationId(66), extent_id: 156, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.383 INFO [2] No action required ReconciliationId(66)
Sep 22 23:22:15.383 DEBG 66 Repair extent 156 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.383 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/09C.copy"
Sep 22 23:22:15.444 INFO accepted connection, remote_addr: 127.0.0.1:59169, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.445 TRCE incoming request, uri: /extent/156/files, method: GET, req_id: 46731d56-3f76-4508-bc3c-d5f9d0869ff2, remote_addr: 127.0.0.1:59169, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.445 INFO request completed, latency_us: 224, response_code: 200, uri: /extent/156/files, method: GET, req_id: 46731d56-3f76-4508-bc3c-d5f9d0869ff2, remote_addr: 127.0.0.1:59169, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.445 INFO eid:156 Found repair files: ["09C", "09C.db"]
Sep 22 23:22:15.445 TRCE incoming request, uri: /newextent/156/data, method: GET, req_id: 1449cc2f-b18c-4016-9559-6085307b4d8c, remote_addr: 127.0.0.1:59169, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.446 INFO request completed, latency_us: 327, response_code: 200, uri: /newextent/156/data, method: GET, req_id: 1449cc2f-b18c-4016-9559-6085307b4d8c, remote_addr: 127.0.0.1:59169, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.450 TRCE incoming request, uri: /newextent/156/db, method: GET, req_id: 8e681b2a-a7b9-4e8f-9583-8241ccec89ec, remote_addr: 127.0.0.1:59169, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.451 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/156/db, method: GET, req_id: 8e681b2a-a7b9-4e8f-9583-8241ccec89ec, remote_addr: 127.0.0.1:59169, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.452 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/09C.copy" to "/tmp/downstairs-zrMnlo6G/00/000/09C.replace"
Sep 22 23:22:15.452 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.453 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/09C.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.453 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09C"
Sep 22 23:22:15.453 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09C.db"
Sep 22 23:22:15.453 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.453 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/09C.replace" to "/tmp/downstairs-zrMnlo6G/00/000/09C.completed"
Sep 22 23:22:15.453 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.453 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.453 DEBG [0] It's time to notify for 66
Sep 22 23:22:15.454 INFO Completion from [0] id:66 status:true
Sep 22 23:22:15.454 INFO [67/752] Repair commands completed
Sep 22 23:22:15.454 INFO Pop front: ReconcileIO { id: ReconciliationId(67), op: ExtentReopen { repair_id: ReconciliationId(67), extent_id: 156 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.454 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.454 INFO [0] received reconcile message
Sep 22 23:22:15.454 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(67), op: ExtentReopen { repair_id: ReconciliationId(67), extent_id: 156 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.454 INFO [0] client ExtentReopen { repair_id: ReconciliationId(67), extent_id: 156 }
Sep 22 23:22:15.454 INFO [1] received reconcile message
Sep 22 23:22:15.454 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(67), op: ExtentReopen { repair_id: ReconciliationId(67), extent_id: 156 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.454 INFO [1] client ExtentReopen { repair_id: ReconciliationId(67), extent_id: 156 }
Sep 22 23:22:15.454 INFO [2] received reconcile message
Sep 22 23:22:15.454 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(67), op: ExtentReopen { repair_id: ReconciliationId(67), extent_id: 156 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.454 INFO [2] client ExtentReopen { repair_id: ReconciliationId(67), extent_id: 156 }
Sep 22 23:22:15.454 DEBG 67 Reopen extent 156
Sep 22 23:22:15.455 DEBG 67 Reopen extent 156
Sep 22 23:22:15.455 DEBG 67 Reopen extent 156
Sep 22 23:22:15.456 DEBG [2] It's time to notify for 67
Sep 22 23:22:15.456 INFO Completion from [2] id:67 status:true
Sep 22 23:22:15.456 INFO [68/752] Repair commands completed
Sep 22 23:22:15.456 INFO Pop front: ReconcileIO { id: ReconciliationId(68), op: ExtentFlush { repair_id: ReconciliationId(68), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.456 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.456 INFO [0] received reconcile message
Sep 22 23:22:15.456 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(68), op: ExtentFlush { repair_id: ReconciliationId(68), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.456 INFO [0] client ExtentFlush { repair_id: ReconciliationId(68), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.456 INFO [1] received reconcile message
Sep 22 23:22:15.456 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(68), op: ExtentFlush { repair_id: ReconciliationId(68), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.456 INFO [1] client ExtentFlush { repair_id: ReconciliationId(68), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.456 INFO [2] received reconcile message
Sep 22 23:22:15.456 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(68), op: ExtentFlush { repair_id: ReconciliationId(68), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.456 INFO [2] client ExtentFlush { repair_id: ReconciliationId(68), extent_id: 125, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.456 DEBG 68 Flush extent 125 with f:2 g:2
Sep 22 23:22:15.456 DEBG Flush just extent 125 with f:2 and g:2
Sep 22 23:22:15.456 DEBG [1] It's time to notify for 68
Sep 22 23:22:15.456 INFO Completion from [1] id:68 status:true
Sep 22 23:22:15.457 INFO [69/752] Repair commands completed
Sep 22 23:22:15.457 INFO Pop front: ReconcileIO { id: ReconciliationId(69), op: ExtentClose { repair_id: ReconciliationId(69), extent_id: 125 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.457 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.457 INFO [0] received reconcile message
Sep 22 23:22:15.457 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(69), op: ExtentClose { repair_id: ReconciliationId(69), extent_id: 125 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.457 INFO [0] client ExtentClose { repair_id: ReconciliationId(69), extent_id: 125 }
Sep 22 23:22:15.457 INFO [1] received reconcile message
Sep 22 23:22:15.457 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(69), op: ExtentClose { repair_id: ReconciliationId(69), extent_id: 125 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.457 INFO [1] client ExtentClose { repair_id: ReconciliationId(69), extent_id: 125 }
Sep 22 23:22:15.457 INFO [2] received reconcile message
Sep 22 23:22:15.457 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(69), op: ExtentClose { repair_id: ReconciliationId(69), extent_id: 125 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.457 INFO [2] client ExtentClose { repair_id: ReconciliationId(69), extent_id: 125 }
Sep 22 23:22:15.457 DEBG 69 Close extent 125
Sep 22 23:22:15.457 DEBG 69 Close extent 125
Sep 22 23:22:15.457 DEBG 69 Close extent 125
Sep 22 23:22:15.458 DEBG [2] It's time to notify for 69
Sep 22 23:22:15.458 INFO Completion from [2] id:69 status:true
Sep 22 23:22:15.458 INFO [70/752] Repair commands completed
Sep 22 23:22:15.458 INFO Pop front: ReconcileIO { id: ReconciliationId(70), op: ExtentRepair { repair_id: ReconciliationId(70), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.458 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.458 INFO [0] received reconcile message
Sep 22 23:22:15.458 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(70), op: ExtentRepair { repair_id: ReconciliationId(70), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.458 INFO [0] client ExtentRepair { repair_id: ReconciliationId(70), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.458 INFO [0] Sending repair request ReconciliationId(70)
Sep 22 23:22:15.458 INFO [1] received reconcile message
Sep 22 23:22:15.458 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(70), op: ExtentRepair { repair_id: ReconciliationId(70), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.458 INFO [1] client ExtentRepair { repair_id: ReconciliationId(70), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.458 INFO [1] No action required ReconciliationId(70)
Sep 22 23:22:15.458 INFO [2] received reconcile message
Sep 22 23:22:15.458 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(70), op: ExtentRepair { repair_id: ReconciliationId(70), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.458 INFO [2] client ExtentRepair { repair_id: ReconciliationId(70), extent_id: 125, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.458 INFO [2] No action required ReconciliationId(70)
Sep 22 23:22:15.458 DEBG 70 Repair extent 125 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.458 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/07D.copy"
Sep 22 23:22:15.524 INFO accepted connection, remote_addr: 127.0.0.1:62531, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.524 TRCE incoming request, uri: /extent/125/files, method: GET, req_id: 3df48713-5d1e-49f8-bf89-93e0af94c2a9, remote_addr: 127.0.0.1:62531, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.525 INFO request completed, latency_us: 214, response_code: 200, uri: /extent/125/files, method: GET, req_id: 3df48713-5d1e-49f8-bf89-93e0af94c2a9, remote_addr: 127.0.0.1:62531, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.525 INFO eid:125 Found repair files: ["07D", "07D.db"]
Sep 22 23:22:15.525 TRCE incoming request, uri: /newextent/125/data, method: GET, req_id: 0317ecf8-f154-4af2-b370-ab8b5b565f14, remote_addr: 127.0.0.1:62531, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.525 INFO request completed, latency_us: 331, response_code: 200, uri: /newextent/125/data, method: GET, req_id: 0317ecf8-f154-4af2-b370-ab8b5b565f14, remote_addr: 127.0.0.1:62531, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.530 TRCE incoming request, uri: /newextent/125/db, method: GET, req_id: 3f7382de-7458-4d96-80ed-e88314d8e5c4, remote_addr: 127.0.0.1:62531, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.530 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/125/db, method: GET, req_id: 3f7382de-7458-4d96-80ed-e88314d8e5c4, remote_addr: 127.0.0.1:62531, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.531 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/07D.copy" to "/tmp/downstairs-zrMnlo6G/00/000/07D.replace"
Sep 22 23:22:15.531 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.532 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/07D.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.533 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07D"
Sep 22 23:22:15.533 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07D.db"
Sep 22 23:22:15.533 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.533 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/07D.replace" to "/tmp/downstairs-zrMnlo6G/00/000/07D.completed"
Sep 22 23:22:15.533 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.533 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.533 DEBG [0] It's time to notify for 70
Sep 22 23:22:15.533 INFO Completion from [0] id:70 status:true
Sep 22 23:22:15.533 INFO [71/752] Repair commands completed
Sep 22 23:22:15.533 INFO Pop front: ReconcileIO { id: ReconciliationId(71), op: ExtentReopen { repair_id: ReconciliationId(71), extent_id: 125 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.533 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.533 INFO [0] received reconcile message
Sep 22 23:22:15.533 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(71), op: ExtentReopen { repair_id: ReconciliationId(71), extent_id: 125 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.533 INFO [0] client ExtentReopen { repair_id: ReconciliationId(71), extent_id: 125 }
Sep 22 23:22:15.533 INFO [1] received reconcile message
Sep 22 23:22:15.533 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(71), op: ExtentReopen { repair_id: ReconciliationId(71), extent_id: 125 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.533 INFO [1] client ExtentReopen { repair_id: ReconciliationId(71), extent_id: 125 }
Sep 22 23:22:15.533 INFO [2] received reconcile message
Sep 22 23:22:15.533 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(71), op: ExtentReopen { repair_id: ReconciliationId(71), extent_id: 125 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.533 INFO [2] client ExtentReopen { repair_id: ReconciliationId(71), extent_id: 125 }
Sep 22 23:22:15.534 DEBG 71 Reopen extent 125
Sep 22 23:22:15.534 DEBG 71 Reopen extent 125
Sep 22 23:22:15.535 DEBG 71 Reopen extent 125
Sep 22 23:22:15.535 DEBG [2] It's time to notify for 71
Sep 22 23:22:15.535 INFO Completion from [2] id:71 status:true
Sep 22 23:22:15.535 INFO [72/752] Repair commands completed
Sep 22 23:22:15.535 INFO Pop front: ReconcileIO { id: ReconciliationId(72), op: ExtentFlush { repair_id: ReconciliationId(72), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.535 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.535 INFO [0] received reconcile message
Sep 22 23:22:15.535 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(72), op: ExtentFlush { repair_id: ReconciliationId(72), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.535 INFO [0] client ExtentFlush { repair_id: ReconciliationId(72), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.535 INFO [1] received reconcile message
Sep 22 23:22:15.535 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(72), op: ExtentFlush { repair_id: ReconciliationId(72), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.535 INFO [1] client ExtentFlush { repair_id: ReconciliationId(72), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.535 INFO [2] received reconcile message
Sep 22 23:22:15.536 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(72), op: ExtentFlush { repair_id: ReconciliationId(72), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.536 INFO [2] client ExtentFlush { repair_id: ReconciliationId(72), extent_id: 37, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.536 DEBG 72 Flush extent 37 with f:2 g:2
Sep 22 23:22:15.536 DEBG Flush just extent 37 with f:2 and g:2
Sep 22 23:22:15.536 DEBG [1] It's time to notify for 72
Sep 22 23:22:15.536 INFO Completion from [1] id:72 status:true
Sep 22 23:22:15.536 INFO [73/752] Repair commands completed
Sep 22 23:22:15.536 INFO Pop front: ReconcileIO { id: ReconciliationId(73), op: ExtentClose { repair_id: ReconciliationId(73), extent_id: 37 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.536 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.536 INFO [0] received reconcile message
Sep 22 23:22:15.536 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(73), op: ExtentClose { repair_id: ReconciliationId(73), extent_id: 37 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.536 INFO [0] client ExtentClose { repair_id: ReconciliationId(73), extent_id: 37 }
Sep 22 23:22:15.536 INFO [1] received reconcile message
Sep 22 23:22:15.536 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(73), op: ExtentClose { repair_id: ReconciliationId(73), extent_id: 37 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.536 INFO [1] client ExtentClose { repair_id: ReconciliationId(73), extent_id: 37 }
Sep 22 23:22:15.536 INFO [2] received reconcile message
Sep 22 23:22:15.536 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(73), op: ExtentClose { repair_id: ReconciliationId(73), extent_id: 37 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.536 INFO [2] client ExtentClose { repair_id: ReconciliationId(73), extent_id: 37 }
Sep 22 23:22:15.536 DEBG 73 Close extent 37
Sep 22 23:22:15.536 DEBG 73 Close extent 37
Sep 22 23:22:15.537 DEBG 73 Close extent 37
Sep 22 23:22:15.537 DEBG [2] It's time to notify for 73
Sep 22 23:22:15.537 INFO Completion from [2] id:73 status:true
Sep 22 23:22:15.537 INFO [74/752] Repair commands completed
Sep 22 23:22:15.537 INFO Pop front: ReconcileIO { id: ReconciliationId(74), op: ExtentRepair { repair_id: ReconciliationId(74), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.537 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.537 INFO [0] received reconcile message
Sep 22 23:22:15.537 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(74), op: ExtentRepair { repair_id: ReconciliationId(74), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.537 INFO [0] client ExtentRepair { repair_id: ReconciliationId(74), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.537 INFO [0] Sending repair request ReconciliationId(74)
Sep 22 23:22:15.537 INFO [1] received reconcile message
Sep 22 23:22:15.537 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(74), op: ExtentRepair { repair_id: ReconciliationId(74), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.538 INFO [1] client ExtentRepair { repair_id: ReconciliationId(74), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.538 INFO [1] No action required ReconciliationId(74)
Sep 22 23:22:15.538 INFO [2] received reconcile message
Sep 22 23:22:15.538 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(74), op: ExtentRepair { repair_id: ReconciliationId(74), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.538 INFO [2] client ExtentRepair { repair_id: ReconciliationId(74), extent_id: 37, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.538 INFO [2] No action required ReconciliationId(74)
Sep 22 23:22:15.538 DEBG 74 Repair extent 37 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.538 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/025.copy"
Sep 22 23:22:15.600 INFO accepted connection, remote_addr: 127.0.0.1:57514, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.600 TRCE incoming request, uri: /extent/37/files, method: GET, req_id: 7af24fd8-4dcb-4498-baed-2b5b2cad8bba, remote_addr: 127.0.0.1:57514, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.601 INFO request completed, latency_us: 200, response_code: 200, uri: /extent/37/files, method: GET, req_id: 7af24fd8-4dcb-4498-baed-2b5b2cad8bba, remote_addr: 127.0.0.1:57514, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.601 INFO eid:37 Found repair files: ["025", "025.db"]
Sep 22 23:22:15.601 TRCE incoming request, uri: /newextent/37/data, method: GET, req_id: fc418cd1-5c07-4cc5-94c5-22717f8bad8d, remote_addr: 127.0.0.1:57514, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.601 INFO request completed, latency_us: 320, response_code: 200, uri: /newextent/37/data, method: GET, req_id: fc418cd1-5c07-4cc5-94c5-22717f8bad8d, remote_addr: 127.0.0.1:57514, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.606 TRCE incoming request, uri: /newextent/37/db, method: GET, req_id: 0be68d3e-964d-47a7-a49b-7dbb2ceddaa8, remote_addr: 127.0.0.1:57514, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.606 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/37/db, method: GET, req_id: 0be68d3e-964d-47a7-a49b-7dbb2ceddaa8, remote_addr: 127.0.0.1:57514, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.607 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/025.copy" to "/tmp/downstairs-zrMnlo6G/00/000/025.replace"
Sep 22 23:22:15.608 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.608 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/025.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.609 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/025"
Sep 22 23:22:15.609 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/025.db"
Sep 22 23:22:15.609 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.609 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/025.replace" to "/tmp/downstairs-zrMnlo6G/00/000/025.completed"
Sep 22 23:22:15.609 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.609 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.609 DEBG [0] It's time to notify for 74
Sep 22 23:22:15.609 INFO Completion from [0] id:74 status:true
Sep 22 23:22:15.609 INFO [75/752] Repair commands completed
Sep 22 23:22:15.609 INFO Pop front: ReconcileIO { id: ReconciliationId(75), op: ExtentReopen { repair_id: ReconciliationId(75), extent_id: 37 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.609 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.609 INFO [0] received reconcile message
Sep 22 23:22:15.609 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(75), op: ExtentReopen { repair_id: ReconciliationId(75), extent_id: 37 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.609 INFO [0] client ExtentReopen { repair_id: ReconciliationId(75), extent_id: 37 }
Sep 22 23:22:15.609 INFO [1] received reconcile message
Sep 22 23:22:15.609 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(75), op: ExtentReopen { repair_id: ReconciliationId(75), extent_id: 37 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.609 INFO [1] client ExtentReopen { repair_id: ReconciliationId(75), extent_id: 37 }
Sep 22 23:22:15.609 INFO [2] received reconcile message
Sep 22 23:22:15.609 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(75), op: ExtentReopen { repair_id: ReconciliationId(75), extent_id: 37 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.609 INFO [2] client ExtentReopen { repair_id: ReconciliationId(75), extent_id: 37 }
Sep 22 23:22:15.610 DEBG 75 Reopen extent 37
Sep 22 23:22:15.610 DEBG 75 Reopen extent 37
Sep 22 23:22:15.611 DEBG 75 Reopen extent 37
Sep 22 23:22:15.611 DEBG [2] It's time to notify for 75
Sep 22 23:22:15.611 INFO Completion from [2] id:75 status:true
Sep 22 23:22:15.611 INFO [76/752] Repair commands completed
Sep 22 23:22:15.611 INFO Pop front: ReconcileIO { id: ReconciliationId(76), op: ExtentFlush { repair_id: ReconciliationId(76), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.611 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.611 INFO [0] received reconcile message
Sep 22 23:22:15.611 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(76), op: ExtentFlush { repair_id: ReconciliationId(76), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.611 INFO [0] client ExtentFlush { repair_id: ReconciliationId(76), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.611 INFO [1] received reconcile message
Sep 22 23:22:15.611 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(76), op: ExtentFlush { repair_id: ReconciliationId(76), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.611 INFO [1] client ExtentFlush { repair_id: ReconciliationId(76), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.612 INFO [2] received reconcile message
Sep 22 23:22:15.612 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(76), op: ExtentFlush { repair_id: ReconciliationId(76), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.612 INFO [2] client ExtentFlush { repair_id: ReconciliationId(76), extent_id: 150, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.612 DEBG 76 Flush extent 150 with f:2 g:2
Sep 22 23:22:15.612 DEBG Flush just extent 150 with f:2 and g:2
Sep 22 23:22:15.612 DEBG [1] It's time to notify for 76
Sep 22 23:22:15.612 INFO Completion from [1] id:76 status:true
Sep 22 23:22:15.612 INFO [77/752] Repair commands completed
Sep 22 23:22:15.612 INFO Pop front: ReconcileIO { id: ReconciliationId(77), op: ExtentClose { repair_id: ReconciliationId(77), extent_id: 150 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.612 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.612 INFO [0] received reconcile message
Sep 22 23:22:15.612 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(77), op: ExtentClose { repair_id: ReconciliationId(77), extent_id: 150 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.612 INFO [0] client ExtentClose { repair_id: ReconciliationId(77), extent_id: 150 }
Sep 22 23:22:15.612 INFO [1] received reconcile message
Sep 22 23:22:15.612 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(77), op: ExtentClose { repair_id: ReconciliationId(77), extent_id: 150 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.612 INFO [1] client ExtentClose { repair_id: ReconciliationId(77), extent_id: 150 }
Sep 22 23:22:15.612 INFO [2] received reconcile message
Sep 22 23:22:15.612 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(77), op: ExtentClose { repair_id: ReconciliationId(77), extent_id: 150 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.612 INFO [2] client ExtentClose { repair_id: ReconciliationId(77), extent_id: 150 }
Sep 22 23:22:15.612 DEBG 77 Close extent 150
Sep 22 23:22:15.613 DEBG 77 Close extent 150
Sep 22 23:22:15.613 DEBG 77 Close extent 150
Sep 22 23:22:15.613 DEBG [2] It's time to notify for 77
Sep 22 23:22:15.613 INFO Completion from [2] id:77 status:true
Sep 22 23:22:15.613 INFO [78/752] Repair commands completed
Sep 22 23:22:15.613 INFO Pop front: ReconcileIO { id: ReconciliationId(78), op: ExtentRepair { repair_id: ReconciliationId(78), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.613 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.613 INFO [0] received reconcile message
Sep 22 23:22:15.613 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(78), op: ExtentRepair { repair_id: ReconciliationId(78), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.613 INFO [0] client ExtentRepair { repair_id: ReconciliationId(78), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.613 INFO [0] Sending repair request ReconciliationId(78)
Sep 22 23:22:15.613 INFO [1] received reconcile message
Sep 22 23:22:15.614 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(78), op: ExtentRepair { repair_id: ReconciliationId(78), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.614 INFO [1] client ExtentRepair { repair_id: ReconciliationId(78), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.614 INFO [1] No action required ReconciliationId(78)
Sep 22 23:22:15.614 INFO [2] received reconcile message
Sep 22 23:22:15.614 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(78), op: ExtentRepair { repair_id: ReconciliationId(78), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.614 INFO [2] client ExtentRepair { repair_id: ReconciliationId(78), extent_id: 150, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.614 INFO [2] No action required ReconciliationId(78)
Sep 22 23:22:15.614 DEBG 78 Repair extent 150 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.614 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/096.copy"
Sep 22 23:22:15.678 INFO accepted connection, remote_addr: 127.0.0.1:45268, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.678 TRCE incoming request, uri: /extent/150/files, method: GET, req_id: e526fbce-6edb-4294-8c12-ba8212f7fad6, remote_addr: 127.0.0.1:45268, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.678 INFO request completed, latency_us: 206, response_code: 200, uri: /extent/150/files, method: GET, req_id: e526fbce-6edb-4294-8c12-ba8212f7fad6, remote_addr: 127.0.0.1:45268, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.678 INFO eid:150 Found repair files: ["096", "096.db"]
Sep 22 23:22:15.679 TRCE incoming request, uri: /newextent/150/data, method: GET, req_id: bc0ecba7-668d-47f5-b3f0-02ead4ae64c8, remote_addr: 127.0.0.1:45268, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.679 INFO request completed, latency_us: 315, response_code: 200, uri: /newextent/150/data, method: GET, req_id: bc0ecba7-668d-47f5-b3f0-02ead4ae64c8, remote_addr: 127.0.0.1:45268, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.684 TRCE incoming request, uri: /newextent/150/db, method: GET, req_id: a38ed99e-fdb0-40d6-af21-1a177c6c6f1d, remote_addr: 127.0.0.1:45268, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.684 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/150/db, method: GET, req_id: a38ed99e-fdb0-40d6-af21-1a177c6c6f1d, remote_addr: 127.0.0.1:45268, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.685 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/096.copy" to "/tmp/downstairs-zrMnlo6G/00/000/096.replace"
Sep 22 23:22:15.685 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.686 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/096.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.686 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/096"
Sep 22 23:22:15.687 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/096.db"
Sep 22 23:22:15.687 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.687 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/096.replace" to "/tmp/downstairs-zrMnlo6G/00/000/096.completed"
Sep 22 23:22:15.687 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.687 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.687 DEBG [0] It's time to notify for 78
Sep 22 23:22:15.687 INFO Completion from [0] id:78 status:true
Sep 22 23:22:15.687 INFO [79/752] Repair commands completed
Sep 22 23:22:15.687 INFO Pop front: ReconcileIO { id: ReconciliationId(79), op: ExtentReopen { repair_id: ReconciliationId(79), extent_id: 150 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.687 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.687 INFO [0] received reconcile message
Sep 22 23:22:15.687 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(79), op: ExtentReopen { repair_id: ReconciliationId(79), extent_id: 150 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.687 INFO [0] client ExtentReopen { repair_id: ReconciliationId(79), extent_id: 150 }
Sep 22 23:22:15.687 INFO [1] received reconcile message
Sep 22 23:22:15.687 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(79), op: ExtentReopen { repair_id: ReconciliationId(79), extent_id: 150 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.687 INFO [1] client ExtentReopen { repair_id: ReconciliationId(79), extent_id: 150 }
Sep 22 23:22:15.687 INFO [2] received reconcile message
Sep 22 23:22:15.687 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(79), op: ExtentReopen { repair_id: ReconciliationId(79), extent_id: 150 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.687 INFO [2] client ExtentReopen { repair_id: ReconciliationId(79), extent_id: 150 }
Sep 22 23:22:15.687 DEBG 79 Reopen extent 150
Sep 22 23:22:15.688 DEBG 79 Reopen extent 150
Sep 22 23:22:15.689 DEBG 79 Reopen extent 150
Sep 22 23:22:15.689 DEBG [2] It's time to notify for 79
Sep 22 23:22:15.689 INFO Completion from [2] id:79 status:true
Sep 22 23:22:15.689 INFO [80/752] Repair commands completed
Sep 22 23:22:15.689 INFO Pop front: ReconcileIO { id: ReconciliationId(80), op: ExtentFlush { repair_id: ReconciliationId(80), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.689 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.689 INFO [0] received reconcile message
Sep 22 23:22:15.689 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(80), op: ExtentFlush { repair_id: ReconciliationId(80), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.689 INFO [0] client ExtentFlush { repair_id: ReconciliationId(80), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.689 INFO [1] received reconcile message
Sep 22 23:22:15.689 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(80), op: ExtentFlush { repair_id: ReconciliationId(80), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.689 INFO [1] client ExtentFlush { repair_id: ReconciliationId(80), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.689 INFO [2] received reconcile message
Sep 22 23:22:15.689 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(80), op: ExtentFlush { repair_id: ReconciliationId(80), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.689 INFO [2] client ExtentFlush { repair_id: ReconciliationId(80), extent_id: 157, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.690 DEBG 80 Flush extent 157 with f:2 g:2
Sep 22 23:22:15.690 DEBG Flush just extent 157 with f:2 and g:2
Sep 22 23:22:15.690 DEBG [1] It's time to notify for 80
Sep 22 23:22:15.690 INFO Completion from [1] id:80 status:true
Sep 22 23:22:15.690 INFO [81/752] Repair commands completed
Sep 22 23:22:15.690 INFO Pop front: ReconcileIO { id: ReconciliationId(81), op: ExtentClose { repair_id: ReconciliationId(81), extent_id: 157 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.690 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.690 INFO [0] received reconcile message
Sep 22 23:22:15.690 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(81), op: ExtentClose { repair_id: ReconciliationId(81), extent_id: 157 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.690 INFO [0] client ExtentClose { repair_id: ReconciliationId(81), extent_id: 157 }
Sep 22 23:22:15.690 INFO [1] received reconcile message
Sep 22 23:22:15.690 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(81), op: ExtentClose { repair_id: ReconciliationId(81), extent_id: 157 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.690 INFO [1] client ExtentClose { repair_id: ReconciliationId(81), extent_id: 157 }
Sep 22 23:22:15.690 INFO [2] received reconcile message
Sep 22 23:22:15.690 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(81), op: ExtentClose { repair_id: ReconciliationId(81), extent_id: 157 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.690 INFO [2] client ExtentClose { repair_id: ReconciliationId(81), extent_id: 157 }
Sep 22 23:22:15.690 DEBG 81 Close extent 157
Sep 22 23:22:15.690 DEBG 81 Close extent 157
Sep 22 23:22:15.691 DEBG 81 Close extent 157
Sep 22 23:22:15.691 DEBG [2] It's time to notify for 81
Sep 22 23:22:15.691 INFO Completion from [2] id:81 status:true
Sep 22 23:22:15.691 INFO [82/752] Repair commands completed
Sep 22 23:22:15.691 INFO Pop front: ReconcileIO { id: ReconciliationId(82), op: ExtentRepair { repair_id: ReconciliationId(82), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.691 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.691 INFO [0] received reconcile message
Sep 22 23:22:15.691 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(82), op: ExtentRepair { repair_id: ReconciliationId(82), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.691 INFO [0] client ExtentRepair { repair_id: ReconciliationId(82), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.691 INFO [0] Sending repair request ReconciliationId(82)
Sep 22 23:22:15.691 INFO [1] received reconcile message
Sep 22 23:22:15.691 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(82), op: ExtentRepair { repair_id: ReconciliationId(82), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.691 INFO [1] client ExtentRepair { repair_id: ReconciliationId(82), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.691 INFO [1] No action required ReconciliationId(82)
Sep 22 23:22:15.692 INFO [2] received reconcile message
Sep 22 23:22:15.692 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(82), op: ExtentRepair { repair_id: ReconciliationId(82), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.692 INFO [2] client ExtentRepair { repair_id: ReconciliationId(82), extent_id: 157, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.692 INFO [2] No action required ReconciliationId(82)
Sep 22 23:22:15.692 DEBG 82 Repair extent 157 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.692 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/09D.copy"
Sep 22 23:22:15.705 ERRO [1] job id 1074 saw error GenericError("test error")
Sep 22 23:22:15.755 INFO accepted connection, remote_addr: 127.0.0.1:64720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.755 TRCE incoming request, uri: /extent/157/files, method: GET, req_id: 8c12b9be-fe98-4557-92f3-e30d68c5feca, remote_addr: 127.0.0.1:64720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.756 INFO request completed, latency_us: 196, response_code: 200, uri: /extent/157/files, method: GET, req_id: 8c12b9be-fe98-4557-92f3-e30d68c5feca, remote_addr: 127.0.0.1:64720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.756 INFO eid:157 Found repair files: ["09D", "09D.db"]
Sep 22 23:22:15.756 TRCE incoming request, uri: /newextent/157/data, method: GET, req_id: 26b5106a-4395-4895-a18c-07c4e90d4627, remote_addr: 127.0.0.1:64720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.756 INFO request completed, latency_us: 321, response_code: 200, uri: /newextent/157/data, method: GET, req_id: 26b5106a-4395-4895-a18c-07c4e90d4627, remote_addr: 127.0.0.1:64720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.761 TRCE incoming request, uri: /newextent/157/db, method: GET, req_id: 9ea01387-2957-4061-b26b-39bfa8708fe1, remote_addr: 127.0.0.1:64720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.761 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/157/db, method: GET, req_id: 9ea01387-2957-4061-b26b-39bfa8708fe1, remote_addr: 127.0.0.1:64720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.762 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/09D.copy" to "/tmp/downstairs-zrMnlo6G/00/000/09D.replace"
Sep 22 23:22:15.762 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.763 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/09D.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.763 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09D"
Sep 22 23:22:15.763 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09D.db"
Sep 22 23:22:15.763 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.763 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/09D.replace" to "/tmp/downstairs-zrMnlo6G/00/000/09D.completed"
Sep 22 23:22:15.763 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.764 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.764 DEBG [0] It's time to notify for 82
Sep 22 23:22:15.764 INFO Completion from [0] id:82 status:true
Sep 22 23:22:15.764 INFO [83/752] Repair commands completed
Sep 22 23:22:15.764 INFO Pop front: ReconcileIO { id: ReconciliationId(83), op: ExtentReopen { repair_id: ReconciliationId(83), extent_id: 157 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.764 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.764 INFO [0] received reconcile message
Sep 22 23:22:15.764 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(83), op: ExtentReopen { repair_id: ReconciliationId(83), extent_id: 157 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.764 INFO [0] client ExtentReopen { repair_id: ReconciliationId(83), extent_id: 157 }
Sep 22 23:22:15.764 INFO [1] received reconcile message
Sep 22 23:22:15.764 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(83), op: ExtentReopen { repair_id: ReconciliationId(83), extent_id: 157 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.764 INFO [1] client ExtentReopen { repair_id: ReconciliationId(83), extent_id: 157 }
Sep 22 23:22:15.764 INFO [2] received reconcile message
Sep 22 23:22:15.764 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(83), op: ExtentReopen { repair_id: ReconciliationId(83), extent_id: 157 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.764 INFO [2] client ExtentReopen { repair_id: ReconciliationId(83), extent_id: 157 }
Sep 22 23:22:15.764 DEBG 83 Reopen extent 157
Sep 22 23:22:15.765 DEBG 83 Reopen extent 157
Sep 22 23:22:15.765 DEBG 83 Reopen extent 157
Sep 22 23:22:15.766 DEBG [2] It's time to notify for 83
Sep 22 23:22:15.766 INFO Completion from [2] id:83 status:true
Sep 22 23:22:15.766 INFO [84/752] Repair commands completed
Sep 22 23:22:15.766 INFO Pop front: ReconcileIO { id: ReconciliationId(84), op: ExtentFlush { repair_id: ReconciliationId(84), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.766 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.766 INFO [0] received reconcile message
Sep 22 23:22:15.766 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(84), op: ExtentFlush { repair_id: ReconciliationId(84), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.766 INFO [0] client ExtentFlush { repair_id: ReconciliationId(84), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.766 INFO [1] received reconcile message
Sep 22 23:22:15.766 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(84), op: ExtentFlush { repair_id: ReconciliationId(84), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.766 INFO [1] client ExtentFlush { repair_id: ReconciliationId(84), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.766 INFO [2] received reconcile message
Sep 22 23:22:15.766 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(84), op: ExtentFlush { repair_id: ReconciliationId(84), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.766 INFO [2] client ExtentFlush { repair_id: ReconciliationId(84), extent_id: 89, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.766 DEBG 84 Flush extent 89 with f:2 g:2
Sep 22 23:22:15.766 DEBG Flush just extent 89 with f:2 and g:2
Sep 22 23:22:15.767 DEBG [1] It's time to notify for 84
Sep 22 23:22:15.767 INFO Completion from [1] id:84 status:true
Sep 22 23:22:15.767 INFO [85/752] Repair commands completed
Sep 22 23:22:15.767 INFO Pop front: ReconcileIO { id: ReconciliationId(85), op: ExtentClose { repair_id: ReconciliationId(85), extent_id: 89 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.767 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.767 INFO [0] received reconcile message
Sep 22 23:22:15.767 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(85), op: ExtentClose { repair_id: ReconciliationId(85), extent_id: 89 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.767 INFO [0] client ExtentClose { repair_id: ReconciliationId(85), extent_id: 89 }
Sep 22 23:22:15.767 INFO [1] received reconcile message
Sep 22 23:22:15.767 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(85), op: ExtentClose { repair_id: ReconciliationId(85), extent_id: 89 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.767 INFO [1] client ExtentClose { repair_id: ReconciliationId(85), extent_id: 89 }
Sep 22 23:22:15.767 INFO [2] received reconcile message
Sep 22 23:22:15.767 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(85), op: ExtentClose { repair_id: ReconciliationId(85), extent_id: 89 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.767 INFO [2] client ExtentClose { repair_id: ReconciliationId(85), extent_id: 89 }
Sep 22 23:22:15.767 DEBG 85 Close extent 89
Sep 22 23:22:15.767 DEBG 85 Close extent 89
Sep 22 23:22:15.768 DEBG 85 Close extent 89
Sep 22 23:22:15.768 DEBG [2] It's time to notify for 85
Sep 22 23:22:15.768 INFO Completion from [2] id:85 status:true
Sep 22 23:22:15.768 INFO [86/752] Repair commands completed
Sep 22 23:22:15.768 INFO Pop front: ReconcileIO { id: ReconciliationId(86), op: ExtentRepair { repair_id: ReconciliationId(86), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.768 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.768 INFO [0] received reconcile message
Sep 22 23:22:15.768 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(86), op: ExtentRepair { repair_id: ReconciliationId(86), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.768 INFO [0] client ExtentRepair { repair_id: ReconciliationId(86), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.768 INFO [0] Sending repair request ReconciliationId(86)
Sep 22 23:22:15.768 INFO [1] received reconcile message
Sep 22 23:22:15.768 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(86), op: ExtentRepair { repair_id: ReconciliationId(86), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.768 INFO [1] client ExtentRepair { repair_id: ReconciliationId(86), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.768 INFO [1] No action required ReconciliationId(86)
Sep 22 23:22:15.768 INFO [2] received reconcile message
Sep 22 23:22:15.768 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(86), op: ExtentRepair { repair_id: ReconciliationId(86), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.768 INFO [2] client ExtentRepair { repair_id: ReconciliationId(86), extent_id: 89, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.768 INFO [2] No action required ReconciliationId(86)
Sep 22 23:22:15.768 DEBG 86 Repair extent 89 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.769 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/059.copy"
Sep 22 23:22:15.832 INFO accepted connection, remote_addr: 127.0.0.1:35193, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.832 TRCE incoming request, uri: /extent/89/files, method: GET, req_id: 53b35606-20fb-47ef-9b2d-0ce135f7e118, remote_addr: 127.0.0.1:35193, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.832 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/89/files, method: GET, req_id: 53b35606-20fb-47ef-9b2d-0ce135f7e118, remote_addr: 127.0.0.1:35193, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.833 INFO eid:89 Found repair files: ["059", "059.db"]
Sep 22 23:22:15.833 TRCE incoming request, uri: /newextent/89/data, method: GET, req_id: 7145ee05-1083-42cb-847d-7f05a954c766, remote_addr: 127.0.0.1:35193, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.833 INFO request completed, latency_us: 261, response_code: 200, uri: /newextent/89/data, method: GET, req_id: 7145ee05-1083-42cb-847d-7f05a954c766, remote_addr: 127.0.0.1:35193, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.838 TRCE incoming request, uri: /newextent/89/db, method: GET, req_id: 85225e57-66b7-429d-b744-19e112df8e3a, remote_addr: 127.0.0.1:35193, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.838 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/89/db, method: GET, req_id: 85225e57-66b7-429d-b744-19e112df8e3a, remote_addr: 127.0.0.1:35193, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.839 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/059.copy" to "/tmp/downstairs-zrMnlo6G/00/000/059.replace"
Sep 22 23:22:15.839 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.840 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/059.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.840 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/059"
Sep 22 23:22:15.841 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/059.db"
Sep 22 23:22:15.841 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.841 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/059.replace" to "/tmp/downstairs-zrMnlo6G/00/000/059.completed"
Sep 22 23:22:15.841 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.841 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.841 DEBG [0] It's time to notify for 86
Sep 22 23:22:15.841 INFO Completion from [0] id:86 status:true
Sep 22 23:22:15.841 INFO [87/752] Repair commands completed
Sep 22 23:22:15.841 INFO Pop front: ReconcileIO { id: ReconciliationId(87), op: ExtentReopen { repair_id: ReconciliationId(87), extent_id: 89 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.841 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.841 INFO [0] received reconcile message
Sep 22 23:22:15.841 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(87), op: ExtentReopen { repair_id: ReconciliationId(87), extent_id: 89 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.841 INFO [0] client ExtentReopen { repair_id: ReconciliationId(87), extent_id: 89 }
Sep 22 23:22:15.841 INFO [1] received reconcile message
Sep 22 23:22:15.841 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(87), op: ExtentReopen { repair_id: ReconciliationId(87), extent_id: 89 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.841 INFO [1] client ExtentReopen { repair_id: ReconciliationId(87), extent_id: 89 }
Sep 22 23:22:15.841 INFO [2] received reconcile message
Sep 22 23:22:15.841 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(87), op: ExtentReopen { repair_id: ReconciliationId(87), extent_id: 89 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.841 INFO [2] client ExtentReopen { repair_id: ReconciliationId(87), extent_id: 89 }
Sep 22 23:22:15.842 DEBG 87 Reopen extent 89
Sep 22 23:22:15.842 DEBG 87 Reopen extent 89
Sep 22 23:22:15.843 DEBG 87 Reopen extent 89
Sep 22 23:22:15.843 DEBG [2] It's time to notify for 87
Sep 22 23:22:15.843 INFO Completion from [2] id:87 status:true
Sep 22 23:22:15.843 INFO [88/752] Repair commands completed
Sep 22 23:22:15.843 INFO Pop front: ReconcileIO { id: ReconciliationId(88), op: ExtentFlush { repair_id: ReconciliationId(88), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.843 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.843 INFO [0] received reconcile message
Sep 22 23:22:15.843 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(88), op: ExtentFlush { repair_id: ReconciliationId(88), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.843 INFO [0] client ExtentFlush { repair_id: ReconciliationId(88), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.843 INFO [1] received reconcile message
Sep 22 23:22:15.843 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(88), op: ExtentFlush { repair_id: ReconciliationId(88), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.843 INFO [1] client ExtentFlush { repair_id: ReconciliationId(88), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.843 INFO [2] received reconcile message
Sep 22 23:22:15.843 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(88), op: ExtentFlush { repair_id: ReconciliationId(88), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.844 INFO [2] client ExtentFlush { repair_id: ReconciliationId(88), extent_id: 163, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.844 DEBG 88 Flush extent 163 with f:2 g:2
Sep 22 23:22:15.844 DEBG Flush just extent 163 with f:2 and g:2
Sep 22 23:22:15.844 DEBG [1] It's time to notify for 88
Sep 22 23:22:15.844 INFO Completion from [1] id:88 status:true
Sep 22 23:22:15.844 INFO [89/752] Repair commands completed
Sep 22 23:22:15.844 INFO Pop front: ReconcileIO { id: ReconciliationId(89), op: ExtentClose { repair_id: ReconciliationId(89), extent_id: 163 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.844 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.844 INFO [0] received reconcile message
Sep 22 23:22:15.844 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(89), op: ExtentClose { repair_id: ReconciliationId(89), extent_id: 163 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.844 INFO [0] client ExtentClose { repair_id: ReconciliationId(89), extent_id: 163 }
Sep 22 23:22:15.844 INFO [1] received reconcile message
Sep 22 23:22:15.844 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(89), op: ExtentClose { repair_id: ReconciliationId(89), extent_id: 163 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.844 INFO [1] client ExtentClose { repair_id: ReconciliationId(89), extent_id: 163 }
Sep 22 23:22:15.844 INFO [2] received reconcile message
Sep 22 23:22:15.844 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(89), op: ExtentClose { repair_id: ReconciliationId(89), extent_id: 163 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.844 INFO [2] client ExtentClose { repair_id: ReconciliationId(89), extent_id: 163 }
Sep 22 23:22:15.844 DEBG 89 Close extent 163
Sep 22 23:22:15.844 DEBG 89 Close extent 163
Sep 22 23:22:15.845 DEBG 89 Close extent 163
Sep 22 23:22:15.845 DEBG [2] It's time to notify for 89
Sep 22 23:22:15.845 INFO Completion from [2] id:89 status:true
Sep 22 23:22:15.845 INFO [90/752] Repair commands completed
Sep 22 23:22:15.845 INFO Pop front: ReconcileIO { id: ReconciliationId(90), op: ExtentRepair { repair_id: ReconciliationId(90), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.845 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.845 INFO [0] received reconcile message
Sep 22 23:22:15.845 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(90), op: ExtentRepair { repair_id: ReconciliationId(90), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.845 INFO [0] client ExtentRepair { repair_id: ReconciliationId(90), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.845 INFO [0] Sending repair request ReconciliationId(90)
Sep 22 23:22:15.845 INFO [1] received reconcile message
Sep 22 23:22:15.845 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(90), op: ExtentRepair { repair_id: ReconciliationId(90), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.845 INFO [1] client ExtentRepair { repair_id: ReconciliationId(90), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.845 INFO [1] No action required ReconciliationId(90)
Sep 22 23:22:15.846 INFO [2] received reconcile message
Sep 22 23:22:15.846 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(90), op: ExtentRepair { repair_id: ReconciliationId(90), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.846 INFO [2] client ExtentRepair { repair_id: ReconciliationId(90), extent_id: 163, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.846 INFO [2] No action required ReconciliationId(90)
Sep 22 23:22:15.846 DEBG 90 Repair extent 163 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.846 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0A3.copy"
Sep 22 23:22:15.911 INFO accepted connection, remote_addr: 127.0.0.1:41776, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.911 TRCE incoming request, uri: /extent/163/files, method: GET, req_id: 4b8df42a-63fb-4714-b9b9-63311dfe1784, remote_addr: 127.0.0.1:41776, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.911 INFO request completed, latency_us: 252, response_code: 200, uri: /extent/163/files, method: GET, req_id: 4b8df42a-63fb-4714-b9b9-63311dfe1784, remote_addr: 127.0.0.1:41776, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.912 INFO eid:163 Found repair files: ["0A3", "0A3.db"]
Sep 22 23:22:15.912 TRCE incoming request, uri: /newextent/163/data, method: GET, req_id: b21bef45-b759-4fbe-94ac-c94f7aa6227f, remote_addr: 127.0.0.1:41776, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.912 INFO request completed, latency_us: 273, response_code: 200, uri: /newextent/163/data, method: GET, req_id: b21bef45-b759-4fbe-94ac-c94f7aa6227f, remote_addr: 127.0.0.1:41776, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.917 TRCE incoming request, uri: /newextent/163/db, method: GET, req_id: fd32cda4-b468-4212-b722-9f9da9789063, remote_addr: 127.0.0.1:41776, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.917 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/163/db, method: GET, req_id: fd32cda4-b468-4212-b722-9f9da9789063, remote_addr: 127.0.0.1:41776, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.918 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0A3.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0A3.replace"
Sep 22 23:22:15.918 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.919 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0A3.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.919 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A3"
Sep 22 23:22:15.920 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A3.db"
Sep 22 23:22:15.920 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.920 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0A3.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0A3.completed"
Sep 22 23:22:15.920 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.920 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.920 DEBG [0] It's time to notify for 90
Sep 22 23:22:15.920 INFO Completion from [0] id:90 status:true
Sep 22 23:22:15.920 INFO [91/752] Repair commands completed
Sep 22 23:22:15.920 INFO Pop front: ReconcileIO { id: ReconciliationId(91), op: ExtentReopen { repair_id: ReconciliationId(91), extent_id: 163 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.920 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.920 INFO [0] received reconcile message
Sep 22 23:22:15.920 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(91), op: ExtentReopen { repair_id: ReconciliationId(91), extent_id: 163 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.920 INFO [0] client ExtentReopen { repair_id: ReconciliationId(91), extent_id: 163 }
Sep 22 23:22:15.920 INFO [1] received reconcile message
Sep 22 23:22:15.920 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(91), op: ExtentReopen { repair_id: ReconciliationId(91), extent_id: 163 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.920 INFO [1] client ExtentReopen { repair_id: ReconciliationId(91), extent_id: 163 }
Sep 22 23:22:15.920 INFO [2] received reconcile message
Sep 22 23:22:15.920 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(91), op: ExtentReopen { repair_id: ReconciliationId(91), extent_id: 163 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.920 INFO [2] client ExtentReopen { repair_id: ReconciliationId(91), extent_id: 163 }
Sep 22 23:22:15.921 DEBG 91 Reopen extent 163
Sep 22 23:22:15.921 DEBG 91 Reopen extent 163
Sep 22 23:22:15.922 DEBG 91 Reopen extent 163
Sep 22 23:22:15.922 DEBG [2] It's time to notify for 91
Sep 22 23:22:15.922 INFO Completion from [2] id:91 status:true
Sep 22 23:22:15.922 INFO [92/752] Repair commands completed
Sep 22 23:22:15.922 INFO Pop front: ReconcileIO { id: ReconciliationId(92), op: ExtentFlush { repair_id: ReconciliationId(92), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.922 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.923 INFO [0] received reconcile message
Sep 22 23:22:15.923 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(92), op: ExtentFlush { repair_id: ReconciliationId(92), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.923 INFO [0] client ExtentFlush { repair_id: ReconciliationId(92), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.923 INFO [1] received reconcile message
Sep 22 23:22:15.923 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(92), op: ExtentFlush { repair_id: ReconciliationId(92), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:15.923 INFO [1] client ExtentFlush { repair_id: ReconciliationId(92), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.923 INFO [2] received reconcile message
Sep 22 23:22:15.923 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(92), op: ExtentFlush { repair_id: ReconciliationId(92), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.923 INFO [2] client ExtentFlush { repair_id: ReconciliationId(92), extent_id: 73, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:15.923 DEBG 92 Flush extent 73 with f:2 g:2
Sep 22 23:22:15.923 DEBG Flush just extent 73 with f:2 and g:2
Sep 22 23:22:15.923 DEBG [1] It's time to notify for 92
Sep 22 23:22:15.923 INFO Completion from [1] id:92 status:true
Sep 22 23:22:15.923 INFO [93/752] Repair commands completed
Sep 22 23:22:15.923 INFO Pop front: ReconcileIO { id: ReconciliationId(93), op: ExtentClose { repair_id: ReconciliationId(93), extent_id: 73 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.923 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.923 INFO [0] received reconcile message
Sep 22 23:22:15.923 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(93), op: ExtentClose { repair_id: ReconciliationId(93), extent_id: 73 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.923 INFO [0] client ExtentClose { repair_id: ReconciliationId(93), extent_id: 73 }
Sep 22 23:22:15.923 INFO [1] received reconcile message
Sep 22 23:22:15.923 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(93), op: ExtentClose { repair_id: ReconciliationId(93), extent_id: 73 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.923 INFO [1] client ExtentClose { repair_id: ReconciliationId(93), extent_id: 73 }
Sep 22 23:22:15.923 INFO [2] received reconcile message
Sep 22 23:22:15.923 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(93), op: ExtentClose { repair_id: ReconciliationId(93), extent_id: 73 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.923 INFO [2] client ExtentClose { repair_id: ReconciliationId(93), extent_id: 73 }
Sep 22 23:22:15.923 DEBG 93 Close extent 73
Sep 22 23:22:15.924 DEBG 93 Close extent 73
Sep 22 23:22:15.924 DEBG 93 Close extent 73
Sep 22 23:22:15.924 DEBG [2] It's time to notify for 93
Sep 22 23:22:15.924 INFO Completion from [2] id:93 status:true
Sep 22 23:22:15.924 INFO [94/752] Repair commands completed
Sep 22 23:22:15.924 INFO Pop front: ReconcileIO { id: ReconciliationId(94), op: ExtentRepair { repair_id: ReconciliationId(94), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.925 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.925 INFO [0] received reconcile message
Sep 22 23:22:15.925 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(94), op: ExtentRepair { repair_id: ReconciliationId(94), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.925 INFO [0] client ExtentRepair { repair_id: ReconciliationId(94), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.925 INFO [0] Sending repair request ReconciliationId(94)
Sep 22 23:22:15.925 INFO [1] received reconcile message
Sep 22 23:22:15.925 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(94), op: ExtentRepair { repair_id: ReconciliationId(94), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.925 INFO [1] client ExtentRepair { repair_id: ReconciliationId(94), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.925 INFO [1] No action required ReconciliationId(94)
Sep 22 23:22:15.925 INFO [2] received reconcile message
Sep 22 23:22:15.925 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(94), op: ExtentRepair { repair_id: ReconciliationId(94), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:15.925 INFO [2] client ExtentRepair { repair_id: ReconciliationId(94), extent_id: 73, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:15.925 INFO [2] No action required ReconciliationId(94)
Sep 22 23:22:15.925 DEBG 94 Repair extent 73 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:15.925 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/049.copy"
Sep 22 23:22:15.988 INFO accepted connection, remote_addr: 127.0.0.1:62296, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.989 TRCE incoming request, uri: /extent/73/files, method: GET, req_id: 8b2b40d1-7e8f-4c9f-ab22-ac6c68c226a6, remote_addr: 127.0.0.1:62296, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.989 INFO request completed, latency_us: 207, response_code: 200, uri: /extent/73/files, method: GET, req_id: 8b2b40d1-7e8f-4c9f-ab22-ac6c68c226a6, remote_addr: 127.0.0.1:62296, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.989 INFO eid:73 Found repair files: ["049", "049.db"]
Sep 22 23:22:15.989 TRCE incoming request, uri: /newextent/73/data, method: GET, req_id: eaef0f02-7fa8-4ef7-b9e6-312d13eb4d7d, remote_addr: 127.0.0.1:62296, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.990 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/73/data, method: GET, req_id: eaef0f02-7fa8-4ef7-b9e6-312d13eb4d7d, remote_addr: 127.0.0.1:62296, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.994 TRCE incoming request, uri: /newextent/73/db, method: GET, req_id: 3457bfc3-78ad-49c3-b233-c0783c537d4f, remote_addr: 127.0.0.1:62296, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.994 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/73/db, method: GET, req_id: 3457bfc3-78ad-49c3-b233-c0783c537d4f, remote_addr: 127.0.0.1:62296, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:15.996 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/049.copy" to "/tmp/downstairs-zrMnlo6G/00/000/049.replace"
Sep 22 23:22:15.996 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.996 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/049.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.997 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/049"
Sep 22 23:22:15.997 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/049.db"
Sep 22 23:22:15.997 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.997 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/049.replace" to "/tmp/downstairs-zrMnlo6G/00/000/049.completed"
Sep 22 23:22:15.997 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.997 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:15.997 DEBG [0] It's time to notify for 94
Sep 22 23:22:15.997 INFO Completion from [0] id:94 status:true
Sep 22 23:22:15.997 INFO [95/752] Repair commands completed
Sep 22 23:22:15.997 INFO Pop front: ReconcileIO { id: ReconciliationId(95), op: ExtentReopen { repair_id: ReconciliationId(95), extent_id: 73 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.997 INFO Sent repair work, now wait for resp
Sep 22 23:22:15.997 INFO [0] received reconcile message
Sep 22 23:22:15.997 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(95), op: ExtentReopen { repair_id: ReconciliationId(95), extent_id: 73 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:15.997 INFO [0] client ExtentReopen { repair_id: ReconciliationId(95), extent_id: 73 }
Sep 22 23:22:15.997 INFO [1] received reconcile message
Sep 22 23:22:15.997 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(95), op: ExtentReopen { repair_id: ReconciliationId(95), extent_id: 73 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:15.997 INFO [1] client ExtentReopen { repair_id: ReconciliationId(95), extent_id: 73 }
Sep 22 23:22:15.997 INFO [2] received reconcile message
Sep 22 23:22:15.998 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(95), op: ExtentReopen { repair_id: ReconciliationId(95), extent_id: 73 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:15.998 INFO [2] client ExtentReopen { repair_id: ReconciliationId(95), extent_id: 73 }
Sep 22 23:22:15.998 DEBG 95 Reopen extent 73
Sep 22 23:22:15.998 DEBG 95 Reopen extent 73
Sep 22 23:22:15.999 DEBG 95 Reopen extent 73
Sep 22 23:22:15.999 DEBG [2] It's time to notify for 95
Sep 22 23:22:15.999 INFO Completion from [2] id:95 status:true
Sep 22 23:22:15.999 INFO [96/752] Repair commands completed
Sep 22 23:22:15.999 INFO Pop front: ReconcileIO { id: ReconciliationId(96), op: ExtentFlush { repair_id: ReconciliationId(96), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:15.999 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.000 INFO [0] received reconcile message
Sep 22 23:22:16.000 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(96), op: ExtentFlush { repair_id: ReconciliationId(96), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.000 INFO [0] client ExtentFlush { repair_id: ReconciliationId(96), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.000 INFO [1] received reconcile message
Sep 22 23:22:16.000 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(96), op: ExtentFlush { repair_id: ReconciliationId(96), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.000 INFO [1] client ExtentFlush { repair_id: ReconciliationId(96), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.000 INFO [2] received reconcile message
Sep 22 23:22:16.000 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(96), op: ExtentFlush { repair_id: ReconciliationId(96), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.000 INFO [2] client ExtentFlush { repair_id: ReconciliationId(96), extent_id: 168, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.000 DEBG 96 Flush extent 168 with f:2 g:2
Sep 22 23:22:16.000 DEBG Flush just extent 168 with f:2 and g:2
Sep 22 23:22:16.000 DEBG [1] It's time to notify for 96
Sep 22 23:22:16.000 INFO Completion from [1] id:96 status:true
Sep 22 23:22:16.000 INFO [97/752] Repair commands completed
Sep 22 23:22:16.000 INFO Pop front: ReconcileIO { id: ReconciliationId(97), op: ExtentClose { repair_id: ReconciliationId(97), extent_id: 168 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.000 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.000 INFO [0] received reconcile message
Sep 22 23:22:16.000 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(97), op: ExtentClose { repair_id: ReconciliationId(97), extent_id: 168 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.000 INFO [0] client ExtentClose { repair_id: ReconciliationId(97), extent_id: 168 }
Sep 22 23:22:16.000 INFO [1] received reconcile message
Sep 22 23:22:16.000 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(97), op: ExtentClose { repair_id: ReconciliationId(97), extent_id: 168 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.000 INFO [1] client ExtentClose { repair_id: ReconciliationId(97), extent_id: 168 }
Sep 22 23:22:16.000 INFO [2] received reconcile message
Sep 22 23:22:16.000 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(97), op: ExtentClose { repair_id: ReconciliationId(97), extent_id: 168 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.000 INFO [2] client ExtentClose { repair_id: ReconciliationId(97), extent_id: 168 }
Sep 22 23:22:16.000 DEBG 97 Close extent 168
Sep 22 23:22:16.001 DEBG 97 Close extent 168
Sep 22 23:22:16.001 DEBG 97 Close extent 168
Sep 22 23:22:16.001 DEBG [2] It's time to notify for 97
Sep 22 23:22:16.001 INFO Completion from [2] id:97 status:true
Sep 22 23:22:16.001 INFO [98/752] Repair commands completed
Sep 22 23:22:16.001 INFO Pop front: ReconcileIO { id: ReconciliationId(98), op: ExtentRepair { repair_id: ReconciliationId(98), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.002 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.002 INFO [0] received reconcile message
Sep 22 23:22:16.002 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(98), op: ExtentRepair { repair_id: ReconciliationId(98), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.002 INFO [0] client ExtentRepair { repair_id: ReconciliationId(98), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.002 INFO [0] Sending repair request ReconciliationId(98)
Sep 22 23:22:16.002 INFO [1] received reconcile message
Sep 22 23:22:16.002 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(98), op: ExtentRepair { repair_id: ReconciliationId(98), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.002 INFO [1] client ExtentRepair { repair_id: ReconciliationId(98), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.002 INFO [1] No action required ReconciliationId(98)
Sep 22 23:22:16.002 INFO [2] received reconcile message
Sep 22 23:22:16.002 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(98), op: ExtentRepair { repair_id: ReconciliationId(98), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.002 INFO [2] client ExtentRepair { repair_id: ReconciliationId(98), extent_id: 168, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.002 INFO [2] No action required ReconciliationId(98)
Sep 22 23:22:16.002 DEBG 98 Repair extent 168 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.002 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0A8.copy"
Sep 22 23:22:16.040 DEBG up_ds_listen was notified
Sep 22 23:22:16.040 DEBG up_ds_listen process 1074
Sep 22 23:22:16.040 DEBG [A] ack job 1074:75, : downstairs
Sep 22 23:22:16.040 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:16.046 DEBG Read      :1075 deps:[JobId(1074)] res:true
Sep 22 23:22:16.066 INFO accepted connection, remote_addr: 127.0.0.1:52304, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.067 TRCE incoming request, uri: /extent/168/files, method: GET, req_id: a13f5a05-99f7-44db-a5d3-d2371a25888f, remote_addr: 127.0.0.1:52304, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.067 INFO request completed, latency_us: 246, response_code: 200, uri: /extent/168/files, method: GET, req_id: a13f5a05-99f7-44db-a5d3-d2371a25888f, remote_addr: 127.0.0.1:52304, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.067 INFO eid:168 Found repair files: ["0A8", "0A8.db"]
Sep 22 23:22:16.068 TRCE incoming request, uri: /newextent/168/data, method: GET, req_id: dae32142-aefa-43dd-8c1a-52908cab0f3c, remote_addr: 127.0.0.1:52304, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.068 INFO request completed, latency_us: 336, response_code: 200, uri: /newextent/168/data, method: GET, req_id: dae32142-aefa-43dd-8c1a-52908cab0f3c, remote_addr: 127.0.0.1:52304, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.070 INFO [lossy] skipping 1076
Sep 22 23:22:16.070 DEBG Flush     :1076 extent_limit None deps:[JobId(1075), JobId(1074)] res:true f:27 g:1
Sep 22 23:22:16.070 INFO [lossy] sleeping 1 second
Sep 22 23:22:16.073 TRCE incoming request, uri: /newextent/168/db, method: GET, req_id: 58bd353f-e1d4-48be-9153-a4df2ce2e901, remote_addr: 127.0.0.1:52304, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.073 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/168/db, method: GET, req_id: 58bd353f-e1d4-48be-9153-a4df2ce2e901, remote_addr: 127.0.0.1:52304, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.074 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0A8.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0A8.replace"
Sep 22 23:22:16.074 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.075 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0A8.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.075 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A8"
Sep 22 23:22:16.075 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A8.db"
Sep 22 23:22:16.075 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.075 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0A8.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0A8.completed"
Sep 22 23:22:16.075 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.075 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.076 DEBG [0] It's time to notify for 98
Sep 22 23:22:16.076 INFO Completion from [0] id:98 status:true
Sep 22 23:22:16.076 INFO [99/752] Repair commands completed
Sep 22 23:22:16.076 INFO Pop front: ReconcileIO { id: ReconciliationId(99), op: ExtentReopen { repair_id: ReconciliationId(99), extent_id: 168 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.076 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.076 INFO [0] received reconcile message
Sep 22 23:22:16.076 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(99), op: ExtentReopen { repair_id: ReconciliationId(99), extent_id: 168 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.076 INFO [0] client ExtentReopen { repair_id: ReconciliationId(99), extent_id: 168 }
Sep 22 23:22:16.076 INFO [1] received reconcile message
Sep 22 23:22:16.076 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(99), op: ExtentReopen { repair_id: ReconciliationId(99), extent_id: 168 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.076 INFO [1] client ExtentReopen { repair_id: ReconciliationId(99), extent_id: 168 }
Sep 22 23:22:16.076 INFO [2] received reconcile message
Sep 22 23:22:16.076 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(99), op: ExtentReopen { repair_id: ReconciliationId(99), extent_id: 168 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.076 INFO [2] client ExtentReopen { repair_id: ReconciliationId(99), extent_id: 168 }
Sep 22 23:22:16.076 DEBG 99 Reopen extent 168
Sep 22 23:22:16.077 DEBG 99 Reopen extent 168
Sep 22 23:22:16.077 DEBG 99 Reopen extent 168
Sep 22 23:22:16.078 DEBG [2] It's time to notify for 99
Sep 22 23:22:16.078 INFO Completion from [2] id:99 status:true
Sep 22 23:22:16.078 INFO [100/752] Repair commands completed
Sep 22 23:22:16.078 INFO Pop front: ReconcileIO { id: ReconciliationId(100), op: ExtentFlush { repair_id: ReconciliationId(100), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.078 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.078 INFO [0] received reconcile message
Sep 22 23:22:16.078 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(100), op: ExtentFlush { repair_id: ReconciliationId(100), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.078 INFO [0] client ExtentFlush { repair_id: ReconciliationId(100), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.078 INFO [1] received reconcile message
Sep 22 23:22:16.078 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(100), op: ExtentFlush { repair_id: ReconciliationId(100), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.078 INFO [1] client ExtentFlush { repair_id: ReconciliationId(100), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.078 INFO [2] received reconcile message
Sep 22 23:22:16.078 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(100), op: ExtentFlush { repair_id: ReconciliationId(100), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.078 INFO [2] client ExtentFlush { repair_id: ReconciliationId(100), extent_id: 176, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.078 DEBG 100 Flush extent 176 with f:2 g:2
Sep 22 23:22:16.078 DEBG Flush just extent 176 with f:2 and g:2
Sep 22 23:22:16.079 DEBG [1] It's time to notify for 100
Sep 22 23:22:16.079 INFO Completion from [1] id:100 status:true
Sep 22 23:22:16.079 INFO [101/752] Repair commands completed
Sep 22 23:22:16.079 INFO Pop front: ReconcileIO { id: ReconciliationId(101), op: ExtentClose { repair_id: ReconciliationId(101), extent_id: 176 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.079 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.079 INFO [0] received reconcile message
Sep 22 23:22:16.079 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(101), op: ExtentClose { repair_id: ReconciliationId(101), extent_id: 176 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.079 INFO [0] client ExtentClose { repair_id: ReconciliationId(101), extent_id: 176 }
Sep 22 23:22:16.079 INFO [1] received reconcile message
Sep 22 23:22:16.079 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(101), op: ExtentClose { repair_id: ReconciliationId(101), extent_id: 176 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.079 INFO [1] client ExtentClose { repair_id: ReconciliationId(101), extent_id: 176 }
Sep 22 23:22:16.079 INFO [2] received reconcile message
Sep 22 23:22:16.079 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(101), op: ExtentClose { repair_id: ReconciliationId(101), extent_id: 176 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.079 INFO [2] client ExtentClose { repair_id: ReconciliationId(101), extent_id: 176 }
Sep 22 23:22:16.079 DEBG 101 Close extent 176
Sep 22 23:22:16.079 DEBG 101 Close extent 176
Sep 22 23:22:16.080 DEBG 101 Close extent 176
Sep 22 23:22:16.080 DEBG [2] It's time to notify for 101
Sep 22 23:22:16.080 INFO Completion from [2] id:101 status:true
Sep 22 23:22:16.080 INFO [102/752] Repair commands completed
Sep 22 23:22:16.080 INFO Pop front: ReconcileIO { id: ReconciliationId(102), op: ExtentRepair { repair_id: ReconciliationId(102), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.080 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.080 INFO [0] received reconcile message
Sep 22 23:22:16.080 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(102), op: ExtentRepair { repair_id: ReconciliationId(102), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.080 INFO [0] client ExtentRepair { repair_id: ReconciliationId(102), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.080 INFO [0] Sending repair request ReconciliationId(102)
Sep 22 23:22:16.080 INFO [1] received reconcile message
Sep 22 23:22:16.080 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(102), op: ExtentRepair { repair_id: ReconciliationId(102), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.080 INFO [1] client ExtentRepair { repair_id: ReconciliationId(102), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.080 INFO [1] No action required ReconciliationId(102)
Sep 22 23:22:16.080 INFO [2] received reconcile message
Sep 22 23:22:16.080 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(102), op: ExtentRepair { repair_id: ReconciliationId(102), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.080 INFO [2] client ExtentRepair { repair_id: ReconciliationId(102), extent_id: 176, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.080 INFO [2] No action required ReconciliationId(102)
Sep 22 23:22:16.081 DEBG 102 Repair extent 176 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.081 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0B0.copy"
Sep 22 23:22:16.142 INFO accepted connection, remote_addr: 127.0.0.1:54002, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.143 TRCE incoming request, uri: /extent/176/files, method: GET, req_id: 2feacb58-ca0f-4033-8e89-e42d03ba85cc, remote_addr: 127.0.0.1:54002, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.143 INFO request completed, latency_us: 208, response_code: 200, uri: /extent/176/files, method: GET, req_id: 2feacb58-ca0f-4033-8e89-e42d03ba85cc, remote_addr: 127.0.0.1:54002, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.143 INFO eid:176 Found repair files: ["0B0", "0B0.db"]
Sep 22 23:22:16.143 TRCE incoming request, uri: /newextent/176/data, method: GET, req_id: 6e17321f-43a4-4a58-8395-2966a1f3fe7c, remote_addr: 127.0.0.1:54002, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.144 INFO request completed, latency_us: 314, response_code: 200, uri: /newextent/176/data, method: GET, req_id: 6e17321f-43a4-4a58-8395-2966a1f3fe7c, remote_addr: 127.0.0.1:54002, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.148 TRCE incoming request, uri: /newextent/176/db, method: GET, req_id: 3b1f43a9-cbad-408b-9dcd-62926c7ce22a, remote_addr: 127.0.0.1:54002, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.149 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/176/db, method: GET, req_id: 3b1f43a9-cbad-408b-9dcd-62926c7ce22a, remote_addr: 127.0.0.1:54002, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.150 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0B0.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0B0.replace"
Sep 22 23:22:16.150 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.151 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0B0.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.151 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B0"
Sep 22 23:22:16.151 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B0.db"
Sep 22 23:22:16.151 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.151 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0B0.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0B0.completed"
Sep 22 23:22:16.151 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.151 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.151 DEBG [0] It's time to notify for 102
Sep 22 23:22:16.151 INFO Completion from [0] id:102 status:true
Sep 22 23:22:16.151 INFO [103/752] Repair commands completed
Sep 22 23:22:16.152 INFO Pop front: ReconcileIO { id: ReconciliationId(103), op: ExtentReopen { repair_id: ReconciliationId(103), extent_id: 176 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.152 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.152 INFO [0] received reconcile message
Sep 22 23:22:16.152 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(103), op: ExtentReopen { repair_id: ReconciliationId(103), extent_id: 176 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.152 INFO [0] client ExtentReopen { repair_id: ReconciliationId(103), extent_id: 176 }
Sep 22 23:22:16.152 INFO [1] received reconcile message
Sep 22 23:22:16.152 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(103), op: ExtentReopen { repair_id: ReconciliationId(103), extent_id: 176 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.152 INFO [1] client ExtentReopen { repair_id: ReconciliationId(103), extent_id: 176 }
Sep 22 23:22:16.152 INFO [2] received reconcile message
Sep 22 23:22:16.152 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(103), op: ExtentReopen { repair_id: ReconciliationId(103), extent_id: 176 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.152 INFO [2] client ExtentReopen { repair_id: ReconciliationId(103), extent_id: 176 }
Sep 22 23:22:16.152 DEBG 103 Reopen extent 176
Sep 22 23:22:16.152 DEBG 103 Reopen extent 176
Sep 22 23:22:16.153 DEBG 103 Reopen extent 176
Sep 22 23:22:16.154 DEBG [2] It's time to notify for 103
Sep 22 23:22:16.154 INFO Completion from [2] id:103 status:true
Sep 22 23:22:16.154 INFO [104/752] Repair commands completed
Sep 22 23:22:16.154 INFO Pop front: ReconcileIO { id: ReconciliationId(104), op: ExtentFlush { repair_id: ReconciliationId(104), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.154 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.154 INFO [0] received reconcile message
Sep 22 23:22:16.154 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(104), op: ExtentFlush { repair_id: ReconciliationId(104), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.154 INFO [0] client ExtentFlush { repair_id: ReconciliationId(104), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.154 INFO [1] received reconcile message
Sep 22 23:22:16.154 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(104), op: ExtentFlush { repair_id: ReconciliationId(104), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.154 INFO [1] client ExtentFlush { repair_id: ReconciliationId(104), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.154 INFO [2] received reconcile message
Sep 22 23:22:16.154 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(104), op: ExtentFlush { repair_id: ReconciliationId(104), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.154 INFO [2] client ExtentFlush { repair_id: ReconciliationId(104), extent_id: 38, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.154 DEBG 104 Flush extent 38 with f:2 g:2
Sep 22 23:22:16.154 DEBG Flush just extent 38 with f:2 and g:2
Sep 22 23:22:16.154 DEBG [1] It's time to notify for 104
Sep 22 23:22:16.154 INFO Completion from [1] id:104 status:true
Sep 22 23:22:16.154 INFO [105/752] Repair commands completed
Sep 22 23:22:16.154 INFO Pop front: ReconcileIO { id: ReconciliationId(105), op: ExtentClose { repair_id: ReconciliationId(105), extent_id: 38 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.154 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.154 INFO [0] received reconcile message
Sep 22 23:22:16.154 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(105), op: ExtentClose { repair_id: ReconciliationId(105), extent_id: 38 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.154 INFO [0] client ExtentClose { repair_id: ReconciliationId(105), extent_id: 38 }
Sep 22 23:22:16.154 INFO [1] received reconcile message
Sep 22 23:22:16.154 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(105), op: ExtentClose { repair_id: ReconciliationId(105), extent_id: 38 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.154 INFO [1] client ExtentClose { repair_id: ReconciliationId(105), extent_id: 38 }
Sep 22 23:22:16.154 INFO [2] received reconcile message
Sep 22 23:22:16.154 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(105), op: ExtentClose { repair_id: ReconciliationId(105), extent_id: 38 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.154 INFO [2] client ExtentClose { repair_id: ReconciliationId(105), extent_id: 38 }
Sep 22 23:22:16.155 DEBG 105 Close extent 38
Sep 22 23:22:16.155 DEBG 105 Close extent 38
Sep 22 23:22:16.155 DEBG 105 Close extent 38
Sep 22 23:22:16.156 DEBG [2] It's time to notify for 105
Sep 22 23:22:16.156 INFO Completion from [2] id:105 status:true
Sep 22 23:22:16.156 INFO [106/752] Repair commands completed
Sep 22 23:22:16.156 INFO Pop front: ReconcileIO { id: ReconciliationId(106), op: ExtentRepair { repair_id: ReconciliationId(106), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.156 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.156 INFO [0] received reconcile message
Sep 22 23:22:16.156 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(106), op: ExtentRepair { repair_id: ReconciliationId(106), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.156 INFO [0] client ExtentRepair { repair_id: ReconciliationId(106), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.156 INFO [0] Sending repair request ReconciliationId(106)
Sep 22 23:22:16.156 INFO [1] received reconcile message
Sep 22 23:22:16.156 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(106), op: ExtentRepair { repair_id: ReconciliationId(106), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.156 INFO [1] client ExtentRepair { repair_id: ReconciliationId(106), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.156 INFO [1] No action required ReconciliationId(106)
Sep 22 23:22:16.156 INFO [2] received reconcile message
Sep 22 23:22:16.156 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(106), op: ExtentRepair { repair_id: ReconciliationId(106), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.156 INFO [2] client ExtentRepair { repair_id: ReconciliationId(106), extent_id: 38, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.156 INFO [2] No action required ReconciliationId(106)
Sep 22 23:22:16.156 DEBG 106 Repair extent 38 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.156 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/026.copy"
Sep 22 23:22:16.221 INFO accepted connection, remote_addr: 127.0.0.1:61720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.221 TRCE incoming request, uri: /extent/38/files, method: GET, req_id: 59e76f2f-61af-4568-8c2e-9fa6edfe72fc, remote_addr: 127.0.0.1:61720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.222 INFO request completed, latency_us: 200, response_code: 200, uri: /extent/38/files, method: GET, req_id: 59e76f2f-61af-4568-8c2e-9fa6edfe72fc, remote_addr: 127.0.0.1:61720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.222 INFO eid:38 Found repair files: ["026", "026.db"]
Sep 22 23:22:16.222 TRCE incoming request, uri: /newextent/38/data, method: GET, req_id: a8b2aab0-e0e8-414f-b5b1-87dec613115c, remote_addr: 127.0.0.1:61720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.223 INFO request completed, latency_us: 320, response_code: 200, uri: /newextent/38/data, method: GET, req_id: a8b2aab0-e0e8-414f-b5b1-87dec613115c, remote_addr: 127.0.0.1:61720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.227 TRCE incoming request, uri: /newextent/38/db, method: GET, req_id: 0ff20c32-2cb7-4e48-9222-59492c0130ac, remote_addr: 127.0.0.1:61720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.228 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/38/db, method: GET, req_id: 0ff20c32-2cb7-4e48-9222-59492c0130ac, remote_addr: 127.0.0.1:61720, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.229 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/026.copy" to "/tmp/downstairs-zrMnlo6G/00/000/026.replace"
Sep 22 23:22:16.229 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.229 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/026.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.230 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/026"
Sep 22 23:22:16.230 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/026.db"
Sep 22 23:22:16.230 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.230 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/026.replace" to "/tmp/downstairs-zrMnlo6G/00/000/026.completed"
Sep 22 23:22:16.230 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.230 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.230 DEBG [0] It's time to notify for 106
Sep 22 23:22:16.230 INFO Completion from [0] id:106 status:true
Sep 22 23:22:16.230 INFO [107/752] Repair commands completed
Sep 22 23:22:16.230 INFO Pop front: ReconcileIO { id: ReconciliationId(107), op: ExtentReopen { repair_id: ReconciliationId(107), extent_id: 38 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.230 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.230 INFO [0] received reconcile message
Sep 22 23:22:16.230 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(107), op: ExtentReopen { repair_id: ReconciliationId(107), extent_id: 38 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.230 INFO [0] client ExtentReopen { repair_id: ReconciliationId(107), extent_id: 38 }
Sep 22 23:22:16.230 INFO [1] received reconcile message
Sep 22 23:22:16.230 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(107), op: ExtentReopen { repair_id: ReconciliationId(107), extent_id: 38 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.231 INFO [1] client ExtentReopen { repair_id: ReconciliationId(107), extent_id: 38 }
Sep 22 23:22:16.231 INFO [2] received reconcile message
Sep 22 23:22:16.231 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(107), op: ExtentReopen { repair_id: ReconciliationId(107), extent_id: 38 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.231 INFO [2] client ExtentReopen { repair_id: ReconciliationId(107), extent_id: 38 }
Sep 22 23:22:16.231 DEBG 107 Reopen extent 38
Sep 22 23:22:16.231 DEBG 107 Reopen extent 38
Sep 22 23:22:16.232 DEBG 107 Reopen extent 38
Sep 22 23:22:16.232 DEBG [2] It's time to notify for 107
Sep 22 23:22:16.232 INFO Completion from [2] id:107 status:true
Sep 22 23:22:16.232 INFO [108/752] Repair commands completed
Sep 22 23:22:16.232 INFO Pop front: ReconcileIO { id: ReconciliationId(108), op: ExtentFlush { repair_id: ReconciliationId(108), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.233 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.233 INFO [0] received reconcile message
Sep 22 23:22:16.233 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(108), op: ExtentFlush { repair_id: ReconciliationId(108), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.233 INFO [0] client ExtentFlush { repair_id: ReconciliationId(108), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.233 INFO [1] received reconcile message
Sep 22 23:22:16.233 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(108), op: ExtentFlush { repair_id: ReconciliationId(108), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.233 INFO [1] client ExtentFlush { repair_id: ReconciliationId(108), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.233 INFO [2] received reconcile message
Sep 22 23:22:16.233 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(108), op: ExtentFlush { repair_id: ReconciliationId(108), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.233 INFO [2] client ExtentFlush { repair_id: ReconciliationId(108), extent_id: 21, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.233 DEBG 108 Flush extent 21 with f:2 g:2
Sep 22 23:22:16.233 DEBG Flush just extent 21 with f:2 and g:2
Sep 22 23:22:16.233 DEBG [1] It's time to notify for 108
Sep 22 23:22:16.233 INFO Completion from [1] id:108 status:true
Sep 22 23:22:16.233 INFO [109/752] Repair commands completed
Sep 22 23:22:16.233 INFO Pop front: ReconcileIO { id: ReconciliationId(109), op: ExtentClose { repair_id: ReconciliationId(109), extent_id: 21 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.233 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.233 INFO [0] received reconcile message
Sep 22 23:22:16.233 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(109), op: ExtentClose { repair_id: ReconciliationId(109), extent_id: 21 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.233 INFO [0] client ExtentClose { repair_id: ReconciliationId(109), extent_id: 21 }
Sep 22 23:22:16.233 INFO [1] received reconcile message
Sep 22 23:22:16.233 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(109), op: ExtentClose { repair_id: ReconciliationId(109), extent_id: 21 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.233 INFO [1] client ExtentClose { repair_id: ReconciliationId(109), extent_id: 21 }
Sep 22 23:22:16.233 INFO [2] received reconcile message
Sep 22 23:22:16.233 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(109), op: ExtentClose { repair_id: ReconciliationId(109), extent_id: 21 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.233 INFO [2] client ExtentClose { repair_id: ReconciliationId(109), extent_id: 21 }
Sep 22 23:22:16.233 DEBG 109 Close extent 21
Sep 22 23:22:16.234 DEBG 109 Close extent 21
Sep 22 23:22:16.234 DEBG 109 Close extent 21
Sep 22 23:22:16.234 DEBG [2] It's time to notify for 109
Sep 22 23:22:16.234 INFO Completion from [2] id:109 status:true
Sep 22 23:22:16.235 INFO [110/752] Repair commands completed
Sep 22 23:22:16.235 INFO Pop front: ReconcileIO { id: ReconciliationId(110), op: ExtentRepair { repair_id: ReconciliationId(110), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.235 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.235 INFO [0] received reconcile message
Sep 22 23:22:16.235 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(110), op: ExtentRepair { repair_id: ReconciliationId(110), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.235 INFO [0] client ExtentRepair { repair_id: ReconciliationId(110), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.235 INFO [0] Sending repair request ReconciliationId(110)
Sep 22 23:22:16.235 INFO [1] received reconcile message
Sep 22 23:22:16.235 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(110), op: ExtentRepair { repair_id: ReconciliationId(110), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.235 INFO [1] client ExtentRepair { repair_id: ReconciliationId(110), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.235 INFO [1] No action required ReconciliationId(110)
Sep 22 23:22:16.235 INFO [2] received reconcile message
Sep 22 23:22:16.235 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(110), op: ExtentRepair { repair_id: ReconciliationId(110), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.235 INFO [2] client ExtentRepair { repair_id: ReconciliationId(110), extent_id: 21, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.235 INFO [2] No action required ReconciliationId(110)
Sep 22 23:22:16.235 DEBG 110 Repair extent 21 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.235 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/015.copy"
Sep 22 23:22:16.299 INFO accepted connection, remote_addr: 127.0.0.1:56752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.299 TRCE incoming request, uri: /extent/21/files, method: GET, req_id: a1161617-249c-4a7b-8faf-8804597a63f8, remote_addr: 127.0.0.1:56752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.299 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/21/files, method: GET, req_id: a1161617-249c-4a7b-8faf-8804597a63f8, remote_addr: 127.0.0.1:56752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.300 INFO eid:21 Found repair files: ["015", "015.db"]
Sep 22 23:22:16.300 TRCE incoming request, uri: /newextent/21/data, method: GET, req_id: f4d1ccfd-95b9-4fe0-a6e0-9e892b2281bc, remote_addr: 127.0.0.1:56752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.300 INFO request completed, latency_us: 260, response_code: 200, uri: /newextent/21/data, method: GET, req_id: f4d1ccfd-95b9-4fe0-a6e0-9e892b2281bc, remote_addr: 127.0.0.1:56752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.305 TRCE incoming request, uri: /newextent/21/db, method: GET, req_id: 5b950338-8fe7-48c8-b12f-f2ed9dd34a82, remote_addr: 127.0.0.1:56752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.305 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/21/db, method: GET, req_id: 5b950338-8fe7-48c8-b12f-f2ed9dd34a82, remote_addr: 127.0.0.1:56752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.306 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/015.copy" to "/tmp/downstairs-zrMnlo6G/00/000/015.replace"
Sep 22 23:22:16.306 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.307 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/015.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.307 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/015"
Sep 22 23:22:16.307 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/015.db"
Sep 22 23:22:16.307 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.307 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/015.replace" to "/tmp/downstairs-zrMnlo6G/00/000/015.completed"
Sep 22 23:22:16.308 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.308 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.308 DEBG [0] It's time to notify for 110
Sep 22 23:22:16.308 INFO Completion from [0] id:110 status:true
Sep 22 23:22:16.308 INFO [111/752] Repair commands completed
Sep 22 23:22:16.308 INFO Pop front: ReconcileIO { id: ReconciliationId(111), op: ExtentReopen { repair_id: ReconciliationId(111), extent_id: 21 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.308 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.308 INFO [0] received reconcile message
Sep 22 23:22:16.308 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(111), op: ExtentReopen { repair_id: ReconciliationId(111), extent_id: 21 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.308 INFO [0] client ExtentReopen { repair_id: ReconciliationId(111), extent_id: 21 }
Sep 22 23:22:16.308 INFO [1] received reconcile message
Sep 22 23:22:16.308 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(111), op: ExtentReopen { repair_id: ReconciliationId(111), extent_id: 21 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.308 INFO [1] client ExtentReopen { repair_id: ReconciliationId(111), extent_id: 21 }
Sep 22 23:22:16.308 INFO [2] received reconcile message
Sep 22 23:22:16.308 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(111), op: ExtentReopen { repair_id: ReconciliationId(111), extent_id: 21 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.308 INFO [2] client ExtentReopen { repair_id: ReconciliationId(111), extent_id: 21 }
Sep 22 23:22:16.308 DEBG 111 Reopen extent 21
Sep 22 23:22:16.309 DEBG 111 Reopen extent 21
Sep 22 23:22:16.309 DEBG 111 Reopen extent 21
Sep 22 23:22:16.310 DEBG [2] It's time to notify for 111
Sep 22 23:22:16.310 INFO Completion from [2] id:111 status:true
Sep 22 23:22:16.310 INFO [112/752] Repair commands completed
Sep 22 23:22:16.310 INFO Pop front: ReconcileIO { id: ReconciliationId(112), op: ExtentFlush { repair_id: ReconciliationId(112), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.310 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.310 INFO [0] received reconcile message
Sep 22 23:22:16.310 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(112), op: ExtentFlush { repair_id: ReconciliationId(112), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.310 INFO [0] client ExtentFlush { repair_id: ReconciliationId(112), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.310 INFO [1] received reconcile message
Sep 22 23:22:16.310 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(112), op: ExtentFlush { repair_id: ReconciliationId(112), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.310 INFO [1] client ExtentFlush { repair_id: ReconciliationId(112), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.310 INFO [2] received reconcile message
Sep 22 23:22:16.310 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(112), op: ExtentFlush { repair_id: ReconciliationId(112), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.310 INFO [2] client ExtentFlush { repair_id: ReconciliationId(112), extent_id: 97, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.310 DEBG 112 Flush extent 97 with f:2 g:2
Sep 22 23:22:16.310 DEBG Flush just extent 97 with f:2 and g:2
Sep 22 23:22:16.311 DEBG [1] It's time to notify for 112
Sep 22 23:22:16.311 INFO Completion from [1] id:112 status:true
Sep 22 23:22:16.311 INFO [113/752] Repair commands completed
Sep 22 23:22:16.311 INFO Pop front: ReconcileIO { id: ReconciliationId(113), op: ExtentClose { repair_id: ReconciliationId(113), extent_id: 97 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.311 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.311 INFO [0] received reconcile message
Sep 22 23:22:16.311 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(113), op: ExtentClose { repair_id: ReconciliationId(113), extent_id: 97 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.311 INFO [0] client ExtentClose { repair_id: ReconciliationId(113), extent_id: 97 }
Sep 22 23:22:16.311 INFO [1] received reconcile message
Sep 22 23:22:16.311 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(113), op: ExtentClose { repair_id: ReconciliationId(113), extent_id: 97 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.311 INFO [1] client ExtentClose { repair_id: ReconciliationId(113), extent_id: 97 }
Sep 22 23:22:16.311 INFO [2] received reconcile message
Sep 22 23:22:16.311 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(113), op: ExtentClose { repair_id: ReconciliationId(113), extent_id: 97 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.311 INFO [2] client ExtentClose { repair_id: ReconciliationId(113), extent_id: 97 }
Sep 22 23:22:16.311 DEBG 113 Close extent 97
Sep 22 23:22:16.311 DEBG 113 Close extent 97
Sep 22 23:22:16.312 DEBG 113 Close extent 97
Sep 22 23:22:16.312 DEBG [2] It's time to notify for 113
Sep 22 23:22:16.312 INFO Completion from [2] id:113 status:true
Sep 22 23:22:16.312 INFO [114/752] Repair commands completed
Sep 22 23:22:16.312 INFO Pop front: ReconcileIO { id: ReconciliationId(114), op: ExtentRepair { repair_id: ReconciliationId(114), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.312 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.312 INFO [0] received reconcile message
Sep 22 23:22:16.312 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(114), op: ExtentRepair { repair_id: ReconciliationId(114), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.312 INFO [0] client ExtentRepair { repair_id: ReconciliationId(114), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.312 INFO [0] Sending repair request ReconciliationId(114)
Sep 22 23:22:16.312 INFO [1] received reconcile message
Sep 22 23:22:16.312 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(114), op: ExtentRepair { repair_id: ReconciliationId(114), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.312 INFO [1] client ExtentRepair { repair_id: ReconciliationId(114), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.312 INFO [1] No action required ReconciliationId(114)
Sep 22 23:22:16.312 INFO [2] received reconcile message
Sep 22 23:22:16.312 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(114), op: ExtentRepair { repair_id: ReconciliationId(114), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.312 INFO [2] client ExtentRepair { repair_id: ReconciliationId(114), extent_id: 97, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.312 INFO [2] No action required ReconciliationId(114)
Sep 22 23:22:16.313 DEBG 114 Repair extent 97 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.313 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/061.copy"
Sep 22 23:22:16.376 INFO accepted connection, remote_addr: 127.0.0.1:40253, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.377 TRCE incoming request, uri: /extent/97/files, method: GET, req_id: 986d2be7-4305-4631-b72d-4b22088e6250, remote_addr: 127.0.0.1:40253, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.377 INFO request completed, latency_us: 268, response_code: 200, uri: /extent/97/files, method: GET, req_id: 986d2be7-4305-4631-b72d-4b22088e6250, remote_addr: 127.0.0.1:40253, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.377 INFO eid:97 Found repair files: ["061", "061.db"]
Sep 22 23:22:16.377 TRCE incoming request, uri: /newextent/97/data, method: GET, req_id: 44d2cbc7-8fa4-41b1-b3aa-78a59b7e8ef3, remote_addr: 127.0.0.1:40253, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.378 INFO request completed, latency_us: 351, response_code: 200, uri: /newextent/97/data, method: GET, req_id: 44d2cbc7-8fa4-41b1-b3aa-78a59b7e8ef3, remote_addr: 127.0.0.1:40253, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.383 TRCE incoming request, uri: /newextent/97/db, method: GET, req_id: 15d9e0f7-fa9d-4c1b-8bdc-a560f9b6cffe, remote_addr: 127.0.0.1:40253, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.383 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/97/db, method: GET, req_id: 15d9e0f7-fa9d-4c1b-8bdc-a560f9b6cffe, remote_addr: 127.0.0.1:40253, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.384 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/061.copy" to "/tmp/downstairs-zrMnlo6G/00/000/061.replace"
Sep 22 23:22:16.384 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.385 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/061.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.386 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/061"
Sep 22 23:22:16.386 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/061.db"
Sep 22 23:22:16.386 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.386 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/061.replace" to "/tmp/downstairs-zrMnlo6G/00/000/061.completed"
Sep 22 23:22:16.386 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.386 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.386 DEBG [0] It's time to notify for 114
Sep 22 23:22:16.386 INFO Completion from [0] id:114 status:true
Sep 22 23:22:16.386 INFO [115/752] Repair commands completed
Sep 22 23:22:16.386 INFO Pop front: ReconcileIO { id: ReconciliationId(115), op: ExtentReopen { repair_id: ReconciliationId(115), extent_id: 97 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.386 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.386 INFO [0] received reconcile message
Sep 22 23:22:16.386 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(115), op: ExtentReopen { repair_id: ReconciliationId(115), extent_id: 97 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.386 INFO [0] client ExtentReopen { repair_id: ReconciliationId(115), extent_id: 97 }
Sep 22 23:22:16.386 INFO [1] received reconcile message
Sep 22 23:22:16.386 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(115), op: ExtentReopen { repair_id: ReconciliationId(115), extent_id: 97 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.386 INFO [1] client ExtentReopen { repair_id: ReconciliationId(115), extent_id: 97 }
Sep 22 23:22:16.386 INFO [2] received reconcile message
Sep 22 23:22:16.387 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(115), op: ExtentReopen { repair_id: ReconciliationId(115), extent_id: 97 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.387 INFO [2] client ExtentReopen { repair_id: ReconciliationId(115), extent_id: 97 }
Sep 22 23:22:16.387 DEBG 115 Reopen extent 97
Sep 22 23:22:16.388 DEBG 115 Reopen extent 97
Sep 22 23:22:16.388 DEBG 115 Reopen extent 97
Sep 22 23:22:16.389 DEBG [2] It's time to notify for 115
Sep 22 23:22:16.389 INFO Completion from [2] id:115 status:true
Sep 22 23:22:16.389 INFO [116/752] Repair commands completed
Sep 22 23:22:16.389 INFO Pop front: ReconcileIO { id: ReconciliationId(116), op: ExtentFlush { repair_id: ReconciliationId(116), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.389 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.389 INFO [0] received reconcile message
Sep 22 23:22:16.389 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(116), op: ExtentFlush { repair_id: ReconciliationId(116), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.389 INFO [0] client ExtentFlush { repair_id: ReconciliationId(116), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.389 INFO [1] received reconcile message
Sep 22 23:22:16.389 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(116), op: ExtentFlush { repair_id: ReconciliationId(116), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.389 INFO [1] client ExtentFlush { repair_id: ReconciliationId(116), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.389 INFO [2] received reconcile message
Sep 22 23:22:16.389 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(116), op: ExtentFlush { repair_id: ReconciliationId(116), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.389 INFO [2] client ExtentFlush { repair_id: ReconciliationId(116), extent_id: 124, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.389 DEBG 116 Flush extent 124 with f:2 g:2
Sep 22 23:22:16.389 DEBG Flush just extent 124 with f:2 and g:2
Sep 22 23:22:16.389 DEBG [1] It's time to notify for 116
Sep 22 23:22:16.389 INFO Completion from [1] id:116 status:true
Sep 22 23:22:16.389 INFO [117/752] Repair commands completed
Sep 22 23:22:16.389 INFO Pop front: ReconcileIO { id: ReconciliationId(117), op: ExtentClose { repair_id: ReconciliationId(117), extent_id: 124 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.389 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.389 INFO [0] received reconcile message
Sep 22 23:22:16.389 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(117), op: ExtentClose { repair_id: ReconciliationId(117), extent_id: 124 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.389 INFO [0] client ExtentClose { repair_id: ReconciliationId(117), extent_id: 124 }
Sep 22 23:22:16.389 INFO [1] received reconcile message
Sep 22 23:22:16.389 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(117), op: ExtentClose { repair_id: ReconciliationId(117), extent_id: 124 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.389 INFO [1] client ExtentClose { repair_id: ReconciliationId(117), extent_id: 124 }
Sep 22 23:22:16.390 INFO [2] received reconcile message
Sep 22 23:22:16.390 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(117), op: ExtentClose { repair_id: ReconciliationId(117), extent_id: 124 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.390 INFO [2] client ExtentClose { repair_id: ReconciliationId(117), extent_id: 124 }
Sep 22 23:22:16.390 DEBG 117 Close extent 124
Sep 22 23:22:16.390 DEBG 117 Close extent 124
Sep 22 23:22:16.390 DEBG 117 Close extent 124
Sep 22 23:22:16.391 DEBG [2] It's time to notify for 117
Sep 22 23:22:16.391 INFO Completion from [2] id:117 status:true
Sep 22 23:22:16.391 INFO [118/752] Repair commands completed
Sep 22 23:22:16.391 INFO Pop front: ReconcileIO { id: ReconciliationId(118), op: ExtentRepair { repair_id: ReconciliationId(118), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.391 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.391 INFO [0] received reconcile message
Sep 22 23:22:16.391 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(118), op: ExtentRepair { repair_id: ReconciliationId(118), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.391 INFO [0] client ExtentRepair { repair_id: ReconciliationId(118), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.391 INFO [0] Sending repair request ReconciliationId(118)
Sep 22 23:22:16.391 INFO [1] received reconcile message
Sep 22 23:22:16.391 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(118), op: ExtentRepair { repair_id: ReconciliationId(118), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.391 INFO [1] client ExtentRepair { repair_id: ReconciliationId(118), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.391 INFO [1] No action required ReconciliationId(118)
Sep 22 23:22:16.391 INFO [2] received reconcile message
Sep 22 23:22:16.391 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(118), op: ExtentRepair { repair_id: ReconciliationId(118), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.391 INFO [2] client ExtentRepair { repair_id: ReconciliationId(118), extent_id: 124, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.391 INFO [2] No action required ReconciliationId(118)
Sep 22 23:22:16.391 DEBG 118 Repair extent 124 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.391 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/07C.copy"
Sep 22 23:22:16.451 DEBG [1] Read AckReady 1075, : downstairs
Sep 22 23:22:16.453 DEBG up_ds_listen was notified
Sep 22 23:22:16.453 DEBG up_ds_listen process 1075
Sep 22 23:22:16.453 DEBG [A] ack job 1075:76, : downstairs
Sep 22 23:22:16.456 INFO accepted connection, remote_addr: 127.0.0.1:43562, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.456 TRCE incoming request, uri: /extent/124/files, method: GET, req_id: 04e2678e-932f-4872-a6de-cf8b8477c475, remote_addr: 127.0.0.1:43562, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.457 INFO request completed, latency_us: 271, response_code: 200, uri: /extent/124/files, method: GET, req_id: 04e2678e-932f-4872-a6de-cf8b8477c475, remote_addr: 127.0.0.1:43562, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.457 INFO eid:124 Found repair files: ["07C", "07C.db"]
Sep 22 23:22:16.457 TRCE incoming request, uri: /newextent/124/data, method: GET, req_id: a9463948-6e86-45c7-a571-55d90fa0b933, remote_addr: 127.0.0.1:43562, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.458 INFO request completed, latency_us: 367, response_code: 200, uri: /newextent/124/data, method: GET, req_id: a9463948-6e86-45c7-a571-55d90fa0b933, remote_addr: 127.0.0.1:43562, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.462 TRCE incoming request, uri: /newextent/124/db, method: GET, req_id: bf2d929e-d96f-45ec-8c18-bd4a2bfa93a3, remote_addr: 127.0.0.1:43562, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.463 INFO request completed, latency_us: 296, response_code: 200, uri: /newextent/124/db, method: GET, req_id: bf2d929e-d96f-45ec-8c18-bd4a2bfa93a3, remote_addr: 127.0.0.1:43562, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.464 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/07C.copy" to "/tmp/downstairs-zrMnlo6G/00/000/07C.replace"
Sep 22 23:22:16.464 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.465 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/07C.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.465 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07C"
Sep 22 23:22:16.465 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07C.db"
Sep 22 23:22:16.465 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.465 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/07C.replace" to "/tmp/downstairs-zrMnlo6G/00/000/07C.completed"
Sep 22 23:22:16.465 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.466 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.466 DEBG [0] It's time to notify for 118
Sep 22 23:22:16.466 INFO Completion from [0] id:118 status:true
Sep 22 23:22:16.466 INFO [119/752] Repair commands completed
Sep 22 23:22:16.466 INFO Pop front: ReconcileIO { id: ReconciliationId(119), op: ExtentReopen { repair_id: ReconciliationId(119), extent_id: 124 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.466 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.466 INFO [0] received reconcile message
Sep 22 23:22:16.466 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(119), op: ExtentReopen { repair_id: ReconciliationId(119), extent_id: 124 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.466 INFO [0] client ExtentReopen { repair_id: ReconciliationId(119), extent_id: 124 }
Sep 22 23:22:16.466 INFO [1] received reconcile message
Sep 22 23:22:16.466 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(119), op: ExtentReopen { repair_id: ReconciliationId(119), extent_id: 124 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.466 INFO [1] client ExtentReopen { repair_id: ReconciliationId(119), extent_id: 124 }
Sep 22 23:22:16.466 INFO [2] received reconcile message
Sep 22 23:22:16.466 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(119), op: ExtentReopen { repair_id: ReconciliationId(119), extent_id: 124 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.466 INFO [2] client ExtentReopen { repair_id: ReconciliationId(119), extent_id: 124 }
Sep 22 23:22:16.466 DEBG 119 Reopen extent 124
Sep 22 23:22:16.467 DEBG 119 Reopen extent 124
Sep 22 23:22:16.468 DEBG 119 Reopen extent 124
Sep 22 23:22:16.468 DEBG [2] It's time to notify for 119
Sep 22 23:22:16.468 INFO Completion from [2] id:119 status:true
Sep 22 23:22:16.468 INFO [120/752] Repair commands completed
Sep 22 23:22:16.468 INFO Pop front: ReconcileIO { id: ReconciliationId(120), op: ExtentFlush { repair_id: ReconciliationId(120), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.468 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.468 INFO [0] received reconcile message
Sep 22 23:22:16.468 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(120), op: ExtentFlush { repair_id: ReconciliationId(120), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.469 INFO [0] client ExtentFlush { repair_id: ReconciliationId(120), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.469 INFO [1] received reconcile message
Sep 22 23:22:16.469 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(120), op: ExtentFlush { repair_id: ReconciliationId(120), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.469 INFO [1] client ExtentFlush { repair_id: ReconciliationId(120), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.469 INFO [2] received reconcile message
Sep 22 23:22:16.469 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(120), op: ExtentFlush { repair_id: ReconciliationId(120), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.469 INFO [2] client ExtentFlush { repair_id: ReconciliationId(120), extent_id: 47, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.469 DEBG 120 Flush extent 47 with f:2 g:2
Sep 22 23:22:16.469 DEBG Flush just extent 47 with f:2 and g:2
Sep 22 23:22:16.469 DEBG [1] It's time to notify for 120
Sep 22 23:22:16.469 INFO Completion from [1] id:120 status:true
Sep 22 23:22:16.469 INFO [121/752] Repair commands completed
Sep 22 23:22:16.469 INFO Pop front: ReconcileIO { id: ReconciliationId(121), op: ExtentClose { repair_id: ReconciliationId(121), extent_id: 47 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.469 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.469 INFO [0] received reconcile message
Sep 22 23:22:16.469 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(121), op: ExtentClose { repair_id: ReconciliationId(121), extent_id: 47 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.469 INFO [0] client ExtentClose { repair_id: ReconciliationId(121), extent_id: 47 }
Sep 22 23:22:16.469 INFO [1] received reconcile message
Sep 22 23:22:16.469 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(121), op: ExtentClose { repair_id: ReconciliationId(121), extent_id: 47 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.469 INFO [1] client ExtentClose { repair_id: ReconciliationId(121), extent_id: 47 }
Sep 22 23:22:16.469 INFO [2] received reconcile message
Sep 22 23:22:16.469 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(121), op: ExtentClose { repair_id: ReconciliationId(121), extent_id: 47 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.469 INFO [2] client ExtentClose { repair_id: ReconciliationId(121), extent_id: 47 }
Sep 22 23:22:16.469 DEBG 121 Close extent 47
Sep 22 23:22:16.470 DEBG 121 Close extent 47
Sep 22 23:22:16.470 DEBG 121 Close extent 47
Sep 22 23:22:16.470 DEBG [2] It's time to notify for 121
Sep 22 23:22:16.470 INFO Completion from [2] id:121 status:true
Sep 22 23:22:16.470 INFO [122/752] Repair commands completed
Sep 22 23:22:16.470 INFO Pop front: ReconcileIO { id: ReconciliationId(122), op: ExtentRepair { repair_id: ReconciliationId(122), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.470 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.471 INFO [0] received reconcile message
Sep 22 23:22:16.471 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(122), op: ExtentRepair { repair_id: ReconciliationId(122), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.471 INFO [0] client ExtentRepair { repair_id: ReconciliationId(122), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.471 INFO [0] Sending repair request ReconciliationId(122)
Sep 22 23:22:16.471 INFO [1] received reconcile message
Sep 22 23:22:16.471 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(122), op: ExtentRepair { repair_id: ReconciliationId(122), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.471 INFO [1] client ExtentRepair { repair_id: ReconciliationId(122), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.471 INFO [1] No action required ReconciliationId(122)
Sep 22 23:22:16.471 INFO [2] received reconcile message
Sep 22 23:22:16.471 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(122), op: ExtentRepair { repair_id: ReconciliationId(122), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.471 INFO [2] client ExtentRepair { repair_id: ReconciliationId(122), extent_id: 47, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.471 INFO [2] No action required ReconciliationId(122)
Sep 22 23:22:16.471 DEBG 122 Repair extent 47 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.471 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/02F.copy"
Sep 22 23:22:16.507 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:16.509 DEBG Flush     :1076 extent_limit None deps:[JobId(1075), JobId(1074)] res:true f:27 g:1
Sep 22 23:22:16.509 INFO [lossy] sleeping 1 second
Sep 22 23:22:16.509 DEBG Flush     :1074 extent_limit None deps:[JobId(1073), JobId(1072)] res:true f:26 g:1
Sep 22 23:22:16.515 DEBG Read      :1075 deps:[JobId(1074)] res:true
Sep 22 23:22:16.533 INFO accepted connection, remote_addr: 127.0.0.1:62204, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.534 TRCE incoming request, uri: /extent/47/files, method: GET, req_id: f2415882-dfa7-4536-81f9-4dfd258cef76, remote_addr: 127.0.0.1:62204, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.534 INFO request completed, latency_us: 213, response_code: 200, uri: /extent/47/files, method: GET, req_id: f2415882-dfa7-4536-81f9-4dfd258cef76, remote_addr: 127.0.0.1:62204, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.534 INFO eid:47 Found repair files: ["02F", "02F.db"]
Sep 22 23:22:16.534 TRCE incoming request, uri: /newextent/47/data, method: GET, req_id: 8bd15551-312d-49de-820c-20975f6aa05c, remote_addr: 127.0.0.1:62204, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.535 INFO request completed, latency_us: 314, response_code: 200, uri: /newextent/47/data, method: GET, req_id: 8bd15551-312d-49de-820c-20975f6aa05c, remote_addr: 127.0.0.1:62204, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.539 TRCE incoming request, uri: /newextent/47/db, method: GET, req_id: f6cd896d-8afa-4468-bf95-d66a7dab6d40, remote_addr: 127.0.0.1:62204, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.540 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/47/db, method: GET, req_id: f6cd896d-8afa-4468-bf95-d66a7dab6d40, remote_addr: 127.0.0.1:62204, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.541 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/02F.copy" to "/tmp/downstairs-zrMnlo6G/00/000/02F.replace"
Sep 22 23:22:16.541 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.542 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/02F.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.542 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02F"
Sep 22 23:22:16.542 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02F.db"
Sep 22 23:22:16.542 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.542 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/02F.replace" to "/tmp/downstairs-zrMnlo6G/00/000/02F.completed"
Sep 22 23:22:16.542 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.542 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.542 DEBG [0] It's time to notify for 122
Sep 22 23:22:16.542 INFO Completion from [0] id:122 status:true
Sep 22 23:22:16.542 INFO [123/752] Repair commands completed
Sep 22 23:22:16.542 INFO Pop front: ReconcileIO { id: ReconciliationId(123), op: ExtentReopen { repair_id: ReconciliationId(123), extent_id: 47 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.542 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.543 INFO [0] received reconcile message
Sep 22 23:22:16.543 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(123), op: ExtentReopen { repair_id: ReconciliationId(123), extent_id: 47 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.543 INFO [0] client ExtentReopen { repair_id: ReconciliationId(123), extent_id: 47 }
Sep 22 23:22:16.543 INFO [1] received reconcile message
Sep 22 23:22:16.543 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(123), op: ExtentReopen { repair_id: ReconciliationId(123), extent_id: 47 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.543 INFO [1] client ExtentReopen { repair_id: ReconciliationId(123), extent_id: 47 }
Sep 22 23:22:16.543 INFO [2] received reconcile message
Sep 22 23:22:16.543 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(123), op: ExtentReopen { repair_id: ReconciliationId(123), extent_id: 47 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.543 INFO [2] client ExtentReopen { repair_id: ReconciliationId(123), extent_id: 47 }
Sep 22 23:22:16.543 DEBG 123 Reopen extent 47
Sep 22 23:22:16.544 DEBG 123 Reopen extent 47
Sep 22 23:22:16.544 DEBG 123 Reopen extent 47
Sep 22 23:22:16.545 DEBG [2] It's time to notify for 123
Sep 22 23:22:16.545 INFO Completion from [2] id:123 status:true
Sep 22 23:22:16.545 INFO [124/752] Repair commands completed
Sep 22 23:22:16.545 INFO Pop front: ReconcileIO { id: ReconciliationId(124), op: ExtentFlush { repair_id: ReconciliationId(124), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.545 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.545 INFO [0] received reconcile message
Sep 22 23:22:16.545 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(124), op: ExtentFlush { repair_id: ReconciliationId(124), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.545 INFO [0] client ExtentFlush { repair_id: ReconciliationId(124), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.545 INFO [1] received reconcile message
Sep 22 23:22:16.545 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(124), op: ExtentFlush { repair_id: ReconciliationId(124), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.545 INFO [1] client ExtentFlush { repair_id: ReconciliationId(124), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.545 INFO [2] received reconcile message
Sep 22 23:22:16.545 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(124), op: ExtentFlush { repair_id: ReconciliationId(124), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.545 INFO [2] client ExtentFlush { repair_id: ReconciliationId(124), extent_id: 179, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.545 DEBG 124 Flush extent 179 with f:2 g:2
Sep 22 23:22:16.545 DEBG Flush just extent 179 with f:2 and g:2
Sep 22 23:22:16.545 DEBG [1] It's time to notify for 124
Sep 22 23:22:16.545 INFO Completion from [1] id:124 status:true
Sep 22 23:22:16.545 INFO [125/752] Repair commands completed
Sep 22 23:22:16.545 INFO Pop front: ReconcileIO { id: ReconciliationId(125), op: ExtentClose { repair_id: ReconciliationId(125), extent_id: 179 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.545 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.545 INFO [0] received reconcile message
Sep 22 23:22:16.545 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(125), op: ExtentClose { repair_id: ReconciliationId(125), extent_id: 179 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.545 INFO [0] client ExtentClose { repair_id: ReconciliationId(125), extent_id: 179 }
Sep 22 23:22:16.545 INFO [1] received reconcile message
Sep 22 23:22:16.545 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(125), op: ExtentClose { repair_id: ReconciliationId(125), extent_id: 179 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.545 INFO [1] client ExtentClose { repair_id: ReconciliationId(125), extent_id: 179 }
Sep 22 23:22:16.546 INFO [2] received reconcile message
Sep 22 23:22:16.546 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(125), op: ExtentClose { repair_id: ReconciliationId(125), extent_id: 179 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.546 INFO [2] client ExtentClose { repair_id: ReconciliationId(125), extent_id: 179 }
Sep 22 23:22:16.546 DEBG 125 Close extent 179
Sep 22 23:22:16.546 DEBG 125 Close extent 179
Sep 22 23:22:16.546 DEBG 125 Close extent 179
Sep 22 23:22:16.547 DEBG [2] It's time to notify for 125
Sep 22 23:22:16.547 INFO Completion from [2] id:125 status:true
Sep 22 23:22:16.547 INFO [126/752] Repair commands completed
Sep 22 23:22:16.547 INFO Pop front: ReconcileIO { id: ReconciliationId(126), op: ExtentRepair { repair_id: ReconciliationId(126), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.547 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.547 INFO [0] received reconcile message
Sep 22 23:22:16.547 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(126), op: ExtentRepair { repair_id: ReconciliationId(126), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.547 INFO [0] client ExtentRepair { repair_id: ReconciliationId(126), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.547 INFO [0] Sending repair request ReconciliationId(126)
Sep 22 23:22:16.547 INFO [1] received reconcile message
Sep 22 23:22:16.547 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(126), op: ExtentRepair { repair_id: ReconciliationId(126), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.547 INFO [1] client ExtentRepair { repair_id: ReconciliationId(126), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.547 INFO [1] No action required ReconciliationId(126)
Sep 22 23:22:16.547 INFO [2] received reconcile message
Sep 22 23:22:16.547 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(126), op: ExtentRepair { repair_id: ReconciliationId(126), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.547 INFO [2] client ExtentRepair { repair_id: ReconciliationId(126), extent_id: 179, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.547 INFO [2] No action required ReconciliationId(126)
Sep 22 23:22:16.547 DEBG 126 Repair extent 179 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.547 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0B3.copy"
Sep 22 23:22:16.586 DEBG IO Read  1077 has deps [JobId(1076)]
Sep 22 23:22:16.611 INFO accepted connection, remote_addr: 127.0.0.1:54083, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.611 TRCE incoming request, uri: /extent/179/files, method: GET, req_id: 3d216e56-1ecf-462b-84bf-1456c9fcf6ed, remote_addr: 127.0.0.1:54083, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.611 INFO request completed, latency_us: 203, response_code: 200, uri: /extent/179/files, method: GET, req_id: 3d216e56-1ecf-462b-84bf-1456c9fcf6ed, remote_addr: 127.0.0.1:54083, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.611 INFO eid:179 Found repair files: ["0B3", "0B3.db"]
Sep 22 23:22:16.612 TRCE incoming request, uri: /newextent/179/data, method: GET, req_id: 40fc9335-ddea-445d-8781-6aa4440b25bd, remote_addr: 127.0.0.1:54083, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.612 INFO request completed, latency_us: 315, response_code: 200, uri: /newextent/179/data, method: GET, req_id: 40fc9335-ddea-445d-8781-6aa4440b25bd, remote_addr: 127.0.0.1:54083, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.617 TRCE incoming request, uri: /newextent/179/db, method: GET, req_id: 8ab3642e-6a6a-448f-97fb-c6540ff59147, remote_addr: 127.0.0.1:54083, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.617 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/179/db, method: GET, req_id: 8ab3642e-6a6a-448f-97fb-c6540ff59147, remote_addr: 127.0.0.1:54083, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.618 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0B3.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0B3.replace"
Sep 22 23:22:16.618 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.619 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0B3.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.619 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B3"
Sep 22 23:22:16.619 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B3.db"
Sep 22 23:22:16.619 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.619 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0B3.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0B3.completed"
Sep 22 23:22:16.619 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.619 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.619 DEBG [0] It's time to notify for 126
Sep 22 23:22:16.620 INFO Completion from [0] id:126 status:true
Sep 22 23:22:16.620 INFO [127/752] Repair commands completed
Sep 22 23:22:16.620 INFO Pop front: ReconcileIO { id: ReconciliationId(127), op: ExtentReopen { repair_id: ReconciliationId(127), extent_id: 179 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.620 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.620 INFO [0] received reconcile message
Sep 22 23:22:16.620 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(127), op: ExtentReopen { repair_id: ReconciliationId(127), extent_id: 179 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.620 INFO [0] client ExtentReopen { repair_id: ReconciliationId(127), extent_id: 179 }
Sep 22 23:22:16.620 INFO [1] received reconcile message
Sep 22 23:22:16.620 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(127), op: ExtentReopen { repair_id: ReconciliationId(127), extent_id: 179 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.620 INFO [1] client ExtentReopen { repair_id: ReconciliationId(127), extent_id: 179 }
Sep 22 23:22:16.620 INFO [2] received reconcile message
Sep 22 23:22:16.620 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(127), op: ExtentReopen { repair_id: ReconciliationId(127), extent_id: 179 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.620 INFO [2] client ExtentReopen { repair_id: ReconciliationId(127), extent_id: 179 }
Sep 22 23:22:16.620 DEBG 127 Reopen extent 179
Sep 22 23:22:16.621 DEBG 127 Reopen extent 179
Sep 22 23:22:16.621 DEBG 127 Reopen extent 179
Sep 22 23:22:16.622 DEBG [2] It's time to notify for 127
Sep 22 23:22:16.622 INFO Completion from [2] id:127 status:true
Sep 22 23:22:16.622 INFO [128/752] Repair commands completed
Sep 22 23:22:16.622 INFO Pop front: ReconcileIO { id: ReconciliationId(128), op: ExtentFlush { repair_id: ReconciliationId(128), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.622 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.622 INFO [0] received reconcile message
Sep 22 23:22:16.622 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(128), op: ExtentFlush { repair_id: ReconciliationId(128), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.622 INFO [0] client ExtentFlush { repair_id: ReconciliationId(128), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.622 INFO [1] received reconcile message
Sep 22 23:22:16.622 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(128), op: ExtentFlush { repair_id: ReconciliationId(128), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.622 INFO [1] client ExtentFlush { repair_id: ReconciliationId(128), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.622 INFO [2] received reconcile message
Sep 22 23:22:16.622 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(128), op: ExtentFlush { repair_id: ReconciliationId(128), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.622 INFO [2] client ExtentFlush { repair_id: ReconciliationId(128), extent_id: 19, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.622 DEBG 128 Flush extent 19 with f:2 g:2
Sep 22 23:22:16.622 DEBG Flush just extent 19 with f:2 and g:2
Sep 22 23:22:16.622 DEBG [1] It's time to notify for 128
Sep 22 23:22:16.622 INFO Completion from [1] id:128 status:true
Sep 22 23:22:16.622 INFO [129/752] Repair commands completed
Sep 22 23:22:16.622 INFO Pop front: ReconcileIO { id: ReconciliationId(129), op: ExtentClose { repair_id: ReconciliationId(129), extent_id: 19 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.622 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.622 INFO [0] received reconcile message
Sep 22 23:22:16.622 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(129), op: ExtentClose { repair_id: ReconciliationId(129), extent_id: 19 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.622 INFO [0] client ExtentClose { repair_id: ReconciliationId(129), extent_id: 19 }
Sep 22 23:22:16.623 INFO [1] received reconcile message
Sep 22 23:22:16.623 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(129), op: ExtentClose { repair_id: ReconciliationId(129), extent_id: 19 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.623 INFO [1] client ExtentClose { repair_id: ReconciliationId(129), extent_id: 19 }
Sep 22 23:22:16.623 INFO [2] received reconcile message
Sep 22 23:22:16.623 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(129), op: ExtentClose { repair_id: ReconciliationId(129), extent_id: 19 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.623 INFO [2] client ExtentClose { repair_id: ReconciliationId(129), extent_id: 19 }
Sep 22 23:22:16.623 DEBG 129 Close extent 19
Sep 22 23:22:16.623 DEBG 129 Close extent 19
Sep 22 23:22:16.623 DEBG 129 Close extent 19
Sep 22 23:22:16.624 DEBG [2] It's time to notify for 129
Sep 22 23:22:16.624 INFO Completion from [2] id:129 status:true
Sep 22 23:22:16.624 INFO [130/752] Repair commands completed
Sep 22 23:22:16.624 INFO Pop front: ReconcileIO { id: ReconciliationId(130), op: ExtentRepair { repair_id: ReconciliationId(130), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.624 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.624 INFO [0] received reconcile message
Sep 22 23:22:16.624 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(130), op: ExtentRepair { repair_id: ReconciliationId(130), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.624 INFO [0] client ExtentRepair { repair_id: ReconciliationId(130), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.624 INFO [0] Sending repair request ReconciliationId(130)
Sep 22 23:22:16.624 INFO [1] received reconcile message
Sep 22 23:22:16.624 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(130), op: ExtentRepair { repair_id: ReconciliationId(130), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.624 INFO [1] client ExtentRepair { repair_id: ReconciliationId(130), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.624 INFO [1] No action required ReconciliationId(130)
Sep 22 23:22:16.624 INFO [2] received reconcile message
Sep 22 23:22:16.624 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(130), op: ExtentRepair { repair_id: ReconciliationId(130), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.624 INFO [2] client ExtentRepair { repair_id: ReconciliationId(130), extent_id: 19, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.624 INFO [2] No action required ReconciliationId(130)
Sep 22 23:22:16.624 DEBG 130 Repair extent 19 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.624 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/013.copy"
Sep 22 23:22:16.689 INFO accepted connection, remote_addr: 127.0.0.1:33589, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.689 TRCE incoming request, uri: /extent/19/files, method: GET, req_id: 531fd6a2-1346-48a5-9a9f-176be177d3a0, remote_addr: 127.0.0.1:33589, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.689 INFO request completed, latency_us: 208, response_code: 200, uri: /extent/19/files, method: GET, req_id: 531fd6a2-1346-48a5-9a9f-176be177d3a0, remote_addr: 127.0.0.1:33589, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.689 INFO eid:19 Found repair files: ["013", "013.db"]
Sep 22 23:22:16.690 TRCE incoming request, uri: /newextent/19/data, method: GET, req_id: b8abf94d-48f5-4db9-bc57-98fbf459827b, remote_addr: 127.0.0.1:33589, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.690 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/19/data, method: GET, req_id: b8abf94d-48f5-4db9-bc57-98fbf459827b, remote_addr: 127.0.0.1:33589, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.695 TRCE incoming request, uri: /newextent/19/db, method: GET, req_id: 1d625ee8-5ebb-4116-846d-673d1ac07c1d, remote_addr: 127.0.0.1:33589, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.695 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/19/db, method: GET, req_id: 1d625ee8-5ebb-4116-846d-673d1ac07c1d, remote_addr: 127.0.0.1:33589, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.696 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/013.copy" to "/tmp/downstairs-zrMnlo6G/00/000/013.replace"
Sep 22 23:22:16.696 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.697 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/013.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.697 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/013"
Sep 22 23:22:16.697 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/013.db"
Sep 22 23:22:16.697 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.697 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/013.replace" to "/tmp/downstairs-zrMnlo6G/00/000/013.completed"
Sep 22 23:22:16.697 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.697 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.698 DEBG [0] It's time to notify for 130
Sep 22 23:22:16.698 INFO Completion from [0] id:130 status:true
Sep 22 23:22:16.698 INFO [131/752] Repair commands completed
Sep 22 23:22:16.698 INFO Pop front: ReconcileIO { id: ReconciliationId(131), op: ExtentReopen { repair_id: ReconciliationId(131), extent_id: 19 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.698 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.698 INFO [0] received reconcile message
Sep 22 23:22:16.698 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(131), op: ExtentReopen { repair_id: ReconciliationId(131), extent_id: 19 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.698 INFO [0] client ExtentReopen { repair_id: ReconciliationId(131), extent_id: 19 }
Sep 22 23:22:16.698 INFO [1] received reconcile message
Sep 22 23:22:16.698 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(131), op: ExtentReopen { repair_id: ReconciliationId(131), extent_id: 19 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.698 INFO [1] client ExtentReopen { repair_id: ReconciliationId(131), extent_id: 19 }
Sep 22 23:22:16.698 INFO [2] received reconcile message
Sep 22 23:22:16.698 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(131), op: ExtentReopen { repair_id: ReconciliationId(131), extent_id: 19 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.698 INFO [2] client ExtentReopen { repair_id: ReconciliationId(131), extent_id: 19 }
Sep 22 23:22:16.698 DEBG 131 Reopen extent 19
Sep 22 23:22:16.699 DEBG 131 Reopen extent 19
Sep 22 23:22:16.699 DEBG 131 Reopen extent 19
Sep 22 23:22:16.700 DEBG [2] It's time to notify for 131
Sep 22 23:22:16.700 INFO Completion from [2] id:131 status:true
Sep 22 23:22:16.700 INFO [132/752] Repair commands completed
Sep 22 23:22:16.700 INFO Pop front: ReconcileIO { id: ReconciliationId(132), op: ExtentFlush { repair_id: ReconciliationId(132), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.700 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.700 INFO [0] received reconcile message
Sep 22 23:22:16.700 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(132), op: ExtentFlush { repair_id: ReconciliationId(132), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.700 INFO [0] client ExtentFlush { repair_id: ReconciliationId(132), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.700 INFO [1] received reconcile message
Sep 22 23:22:16.700 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(132), op: ExtentFlush { repair_id: ReconciliationId(132), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.700 INFO [1] client ExtentFlush { repair_id: ReconciliationId(132), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.700 INFO [2] received reconcile message
Sep 22 23:22:16.700 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(132), op: ExtentFlush { repair_id: ReconciliationId(132), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.700 INFO [2] client ExtentFlush { repair_id: ReconciliationId(132), extent_id: 167, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.700 DEBG 132 Flush extent 167 with f:2 g:2
Sep 22 23:22:16.700 DEBG Flush just extent 167 with f:2 and g:2
Sep 22 23:22:16.701 DEBG [1] It's time to notify for 132
Sep 22 23:22:16.701 INFO Completion from [1] id:132 status:true
Sep 22 23:22:16.701 INFO [133/752] Repair commands completed
Sep 22 23:22:16.701 INFO Pop front: ReconcileIO { id: ReconciliationId(133), op: ExtentClose { repair_id: ReconciliationId(133), extent_id: 167 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.701 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.701 INFO [0] received reconcile message
Sep 22 23:22:16.701 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(133), op: ExtentClose { repair_id: ReconciliationId(133), extent_id: 167 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.701 INFO [0] client ExtentClose { repair_id: ReconciliationId(133), extent_id: 167 }
Sep 22 23:22:16.701 INFO [1] received reconcile message
Sep 22 23:22:16.701 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(133), op: ExtentClose { repair_id: ReconciliationId(133), extent_id: 167 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.701 INFO [1] client ExtentClose { repair_id: ReconciliationId(133), extent_id: 167 }
Sep 22 23:22:16.701 INFO [2] received reconcile message
Sep 22 23:22:16.701 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(133), op: ExtentClose { repair_id: ReconciliationId(133), extent_id: 167 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.701 INFO [2] client ExtentClose { repair_id: ReconciliationId(133), extent_id: 167 }
Sep 22 23:22:16.701 DEBG 133 Close extent 167
Sep 22 23:22:16.701 DEBG 133 Close extent 167
Sep 22 23:22:16.702 DEBG 133 Close extent 167
Sep 22 23:22:16.702 DEBG [2] It's time to notify for 133
Sep 22 23:22:16.702 INFO Completion from [2] id:133 status:true
Sep 22 23:22:16.702 INFO [134/752] Repair commands completed
Sep 22 23:22:16.702 INFO Pop front: ReconcileIO { id: ReconciliationId(134), op: ExtentRepair { repair_id: ReconciliationId(134), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.702 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.702 INFO [0] received reconcile message
Sep 22 23:22:16.702 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(134), op: ExtentRepair { repair_id: ReconciliationId(134), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.702 INFO [0] client ExtentRepair { repair_id: ReconciliationId(134), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.702 INFO [0] Sending repair request ReconciliationId(134)
Sep 22 23:22:16.702 INFO [1] received reconcile message
Sep 22 23:22:16.702 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(134), op: ExtentRepair { repair_id: ReconciliationId(134), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.702 INFO [1] client ExtentRepair { repair_id: ReconciliationId(134), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.702 INFO [1] No action required ReconciliationId(134)
Sep 22 23:22:16.702 INFO [2] received reconcile message
Sep 22 23:22:16.702 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(134), op: ExtentRepair { repair_id: ReconciliationId(134), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.702 INFO [2] client ExtentRepair { repair_id: ReconciliationId(134), extent_id: 167, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.702 INFO [2] No action required ReconciliationId(134)
Sep 22 23:22:16.703 DEBG 134 Repair extent 167 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.703 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0A7.copy"
Sep 22 23:22:16.766 INFO accepted connection, remote_addr: 127.0.0.1:58639, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.766 TRCE incoming request, uri: /extent/167/files, method: GET, req_id: a928b88a-8d6d-4263-93d2-e7b6b06142c3, remote_addr: 127.0.0.1:58639, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.766 INFO request completed, latency_us: 220, response_code: 200, uri: /extent/167/files, method: GET, req_id: a928b88a-8d6d-4263-93d2-e7b6b06142c3, remote_addr: 127.0.0.1:58639, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.766 INFO eid:167 Found repair files: ["0A7", "0A7.db"]
Sep 22 23:22:16.767 TRCE incoming request, uri: /newextent/167/data, method: GET, req_id: 8be497f6-c334-4224-ad8c-ce3e63dc02a0, remote_addr: 127.0.0.1:58639, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.767 INFO request completed, latency_us: 315, response_code: 200, uri: /newextent/167/data, method: GET, req_id: 8be497f6-c334-4224-ad8c-ce3e63dc02a0, remote_addr: 127.0.0.1:58639, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.772 TRCE incoming request, uri: /newextent/167/db, method: GET, req_id: 1619eb67-fadf-4829-9f8a-904e0bdd533e, remote_addr: 127.0.0.1:58639, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.772 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/167/db, method: GET, req_id: 1619eb67-fadf-4829-9f8a-904e0bdd533e, remote_addr: 127.0.0.1:58639, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.773 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0A7.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0A7.replace"
Sep 22 23:22:16.773 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.774 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0A7.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.774 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A7"
Sep 22 23:22:16.774 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A7.db"
Sep 22 23:22:16.774 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.774 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0A7.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0A7.completed"
Sep 22 23:22:16.774 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.774 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.774 DEBG [0] It's time to notify for 134
Sep 22 23:22:16.775 INFO Completion from [0] id:134 status:true
Sep 22 23:22:16.775 INFO [135/752] Repair commands completed
Sep 22 23:22:16.775 INFO Pop front: ReconcileIO { id: ReconciliationId(135), op: ExtentReopen { repair_id: ReconciliationId(135), extent_id: 167 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.775 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.775 INFO [0] received reconcile message
Sep 22 23:22:16.775 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(135), op: ExtentReopen { repair_id: ReconciliationId(135), extent_id: 167 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.775 INFO [0] client ExtentReopen { repair_id: ReconciliationId(135), extent_id: 167 }
Sep 22 23:22:16.775 INFO [1] received reconcile message
Sep 22 23:22:16.775 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(135), op: ExtentReopen { repair_id: ReconciliationId(135), extent_id: 167 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.775 INFO [1] client ExtentReopen { repair_id: ReconciliationId(135), extent_id: 167 }
Sep 22 23:22:16.775 INFO [2] received reconcile message
Sep 22 23:22:16.775 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(135), op: ExtentReopen { repair_id: ReconciliationId(135), extent_id: 167 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.775 INFO [2] client ExtentReopen { repair_id: ReconciliationId(135), extent_id: 167 }
Sep 22 23:22:16.775 DEBG 135 Reopen extent 167
Sep 22 23:22:16.776 DEBG 135 Reopen extent 167
Sep 22 23:22:16.776 DEBG 135 Reopen extent 167
Sep 22 23:22:16.777 DEBG [2] It's time to notify for 135
Sep 22 23:22:16.777 INFO Completion from [2] id:135 status:true
Sep 22 23:22:16.777 INFO [136/752] Repair commands completed
Sep 22 23:22:16.777 INFO Pop front: ReconcileIO { id: ReconciliationId(136), op: ExtentFlush { repair_id: ReconciliationId(136), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.777 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.777 INFO [0] received reconcile message
Sep 22 23:22:16.777 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(136), op: ExtentFlush { repair_id: ReconciliationId(136), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.777 INFO [0] client ExtentFlush { repair_id: ReconciliationId(136), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.777 INFO [1] received reconcile message
Sep 22 23:22:16.777 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(136), op: ExtentFlush { repair_id: ReconciliationId(136), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.777 INFO [1] client ExtentFlush { repair_id: ReconciliationId(136), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.777 INFO [2] received reconcile message
Sep 22 23:22:16.777 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(136), op: ExtentFlush { repair_id: ReconciliationId(136), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.777 INFO [2] client ExtentFlush { repair_id: ReconciliationId(136), extent_id: 182, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.777 DEBG 136 Flush extent 182 with f:2 g:2
Sep 22 23:22:16.777 DEBG Flush just extent 182 with f:2 and g:2
Sep 22 23:22:16.777 DEBG [1] It's time to notify for 136
Sep 22 23:22:16.777 INFO Completion from [1] id:136 status:true
Sep 22 23:22:16.777 INFO [137/752] Repair commands completed
Sep 22 23:22:16.777 INFO Pop front: ReconcileIO { id: ReconciliationId(137), op: ExtentClose { repair_id: ReconciliationId(137), extent_id: 182 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.777 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.777 INFO [0] received reconcile message
Sep 22 23:22:16.777 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(137), op: ExtentClose { repair_id: ReconciliationId(137), extent_id: 182 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.777 INFO [0] client ExtentClose { repair_id: ReconciliationId(137), extent_id: 182 }
Sep 22 23:22:16.778 INFO [1] received reconcile message
Sep 22 23:22:16.778 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(137), op: ExtentClose { repair_id: ReconciliationId(137), extent_id: 182 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.778 INFO [1] client ExtentClose { repair_id: ReconciliationId(137), extent_id: 182 }
Sep 22 23:22:16.778 INFO [2] received reconcile message
Sep 22 23:22:16.778 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(137), op: ExtentClose { repair_id: ReconciliationId(137), extent_id: 182 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.778 INFO [2] client ExtentClose { repair_id: ReconciliationId(137), extent_id: 182 }
Sep 22 23:22:16.778 DEBG 137 Close extent 182
Sep 22 23:22:16.778 DEBG 137 Close extent 182
Sep 22 23:22:16.778 DEBG 137 Close extent 182
Sep 22 23:22:16.779 DEBG [2] It's time to notify for 137
Sep 22 23:22:16.779 INFO Completion from [2] id:137 status:true
Sep 22 23:22:16.779 INFO [138/752] Repair commands completed
Sep 22 23:22:16.779 INFO Pop front: ReconcileIO { id: ReconciliationId(138), op: ExtentRepair { repair_id: ReconciliationId(138), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.779 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.779 INFO [0] received reconcile message
Sep 22 23:22:16.779 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(138), op: ExtentRepair { repair_id: ReconciliationId(138), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.779 INFO [0] client ExtentRepair { repair_id: ReconciliationId(138), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.779 INFO [0] Sending repair request ReconciliationId(138)
Sep 22 23:22:16.779 INFO [1] received reconcile message
Sep 22 23:22:16.779 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(138), op: ExtentRepair { repair_id: ReconciliationId(138), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.779 INFO [1] client ExtentRepair { repair_id: ReconciliationId(138), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.779 INFO [1] No action required ReconciliationId(138)
Sep 22 23:22:16.779 INFO [2] received reconcile message
Sep 22 23:22:16.779 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(138), op: ExtentRepair { repair_id: ReconciliationId(138), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.779 INFO [2] client ExtentRepair { repair_id: ReconciliationId(138), extent_id: 182, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.779 INFO [2] No action required ReconciliationId(138)
Sep 22 23:22:16.779 DEBG 138 Repair extent 182 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.779 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0B6.copy"
Sep 22 23:22:16.843 INFO accepted connection, remote_addr: 127.0.0.1:41659, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.843 TRCE incoming request, uri: /extent/182/files, method: GET, req_id: 61175ef0-46f1-4596-8edc-fd061f1ed308, remote_addr: 127.0.0.1:41659, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.844 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/182/files, method: GET, req_id: 61175ef0-46f1-4596-8edc-fd061f1ed308, remote_addr: 127.0.0.1:41659, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.844 INFO eid:182 Found repair files: ["0B6", "0B6.db"]
Sep 22 23:22:16.844 TRCE incoming request, uri: /newextent/182/data, method: GET, req_id: aa3cf251-40e5-4122-a2e5-d277bc19b9ec, remote_addr: 127.0.0.1:41659, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.844 INFO request completed, latency_us: 319, response_code: 200, uri: /newextent/182/data, method: GET, req_id: aa3cf251-40e5-4122-a2e5-d277bc19b9ec, remote_addr: 127.0.0.1:41659, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.849 TRCE incoming request, uri: /newextent/182/db, method: GET, req_id: a8840280-cf29-48c8-809c-40df6e0275eb, remote_addr: 127.0.0.1:41659, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.850 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/182/db, method: GET, req_id: a8840280-cf29-48c8-809c-40df6e0275eb, remote_addr: 127.0.0.1:41659, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.851 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0B6.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0B6.replace"
Sep 22 23:22:16.851 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.852 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0B6.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.852 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B6"
Sep 22 23:22:16.852 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B6.db"
Sep 22 23:22:16.852 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.852 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0B6.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0B6.completed"
Sep 22 23:22:16.852 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.852 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.852 DEBG [0] It's time to notify for 138
Sep 22 23:22:16.852 INFO Completion from [0] id:138 status:true
Sep 22 23:22:16.852 INFO [139/752] Repair commands completed
Sep 22 23:22:16.852 INFO Pop front: ReconcileIO { id: ReconciliationId(139), op: ExtentReopen { repair_id: ReconciliationId(139), extent_id: 182 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.852 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.852 INFO [0] received reconcile message
Sep 22 23:22:16.853 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(139), op: ExtentReopen { repair_id: ReconciliationId(139), extent_id: 182 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.853 INFO [0] client ExtentReopen { repair_id: ReconciliationId(139), extent_id: 182 }
Sep 22 23:22:16.853 INFO [1] received reconcile message
Sep 22 23:22:16.853 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(139), op: ExtentReopen { repair_id: ReconciliationId(139), extent_id: 182 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.853 INFO [1] client ExtentReopen { repair_id: ReconciliationId(139), extent_id: 182 }
Sep 22 23:22:16.853 INFO [2] received reconcile message
Sep 22 23:22:16.853 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(139), op: ExtentReopen { repair_id: ReconciliationId(139), extent_id: 182 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.853 INFO [2] client ExtentReopen { repair_id: ReconciliationId(139), extent_id: 182 }
Sep 22 23:22:16.853 DEBG 139 Reopen extent 182
Sep 22 23:22:16.853 DEBG 139 Reopen extent 182
Sep 22 23:22:16.854 DEBG 139 Reopen extent 182
Sep 22 23:22:16.854 DEBG [2] It's time to notify for 139
Sep 22 23:22:16.855 INFO Completion from [2] id:139 status:true
Sep 22 23:22:16.855 INFO [140/752] Repair commands completed
Sep 22 23:22:16.855 INFO Pop front: ReconcileIO { id: ReconciliationId(140), op: ExtentFlush { repair_id: ReconciliationId(140), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.855 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.855 INFO [0] received reconcile message
Sep 22 23:22:16.855 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(140), op: ExtentFlush { repair_id: ReconciliationId(140), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.855 INFO [0] client ExtentFlush { repair_id: ReconciliationId(140), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.855 INFO [1] received reconcile message
Sep 22 23:22:16.855 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(140), op: ExtentFlush { repair_id: ReconciliationId(140), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.855 INFO [1] client ExtentFlush { repair_id: ReconciliationId(140), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.855 INFO [2] received reconcile message
Sep 22 23:22:16.855 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(140), op: ExtentFlush { repair_id: ReconciliationId(140), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.855 INFO [2] client ExtentFlush { repair_id: ReconciliationId(140), extent_id: 78, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.855 DEBG 140 Flush extent 78 with f:2 g:2
Sep 22 23:22:16.855 DEBG Flush just extent 78 with f:2 and g:2
Sep 22 23:22:16.855 DEBG [1] It's time to notify for 140
Sep 22 23:22:16.855 INFO Completion from [1] id:140 status:true
Sep 22 23:22:16.855 INFO [141/752] Repair commands completed
Sep 22 23:22:16.855 INFO Pop front: ReconcileIO { id: ReconciliationId(141), op: ExtentClose { repair_id: ReconciliationId(141), extent_id: 78 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.855 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.855 INFO [0] received reconcile message
Sep 22 23:22:16.855 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(141), op: ExtentClose { repair_id: ReconciliationId(141), extent_id: 78 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.855 INFO [0] client ExtentClose { repair_id: ReconciliationId(141), extent_id: 78 }
Sep 22 23:22:16.855 INFO [1] received reconcile message
Sep 22 23:22:16.855 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(141), op: ExtentClose { repair_id: ReconciliationId(141), extent_id: 78 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.855 INFO [1] client ExtentClose { repair_id: ReconciliationId(141), extent_id: 78 }
Sep 22 23:22:16.855 INFO [2] received reconcile message
Sep 22 23:22:16.855 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(141), op: ExtentClose { repair_id: ReconciliationId(141), extent_id: 78 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.855 INFO [2] client ExtentClose { repair_id: ReconciliationId(141), extent_id: 78 }
Sep 22 23:22:16.856 DEBG 141 Close extent 78
Sep 22 23:22:16.856 DEBG 141 Close extent 78
Sep 22 23:22:16.856 DEBG 141 Close extent 78
Sep 22 23:22:16.856 DEBG [2] It's time to notify for 141
Sep 22 23:22:16.857 INFO Completion from [2] id:141 status:true
Sep 22 23:22:16.857 INFO [142/752] Repair commands completed
Sep 22 23:22:16.857 INFO Pop front: ReconcileIO { id: ReconciliationId(142), op: ExtentRepair { repair_id: ReconciliationId(142), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.857 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.857 INFO [0] received reconcile message
Sep 22 23:22:16.857 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(142), op: ExtentRepair { repair_id: ReconciliationId(142), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.857 INFO [0] client ExtentRepair { repair_id: ReconciliationId(142), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.857 INFO [0] Sending repair request ReconciliationId(142)
Sep 22 23:22:16.857 INFO [1] received reconcile message
Sep 22 23:22:16.857 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(142), op: ExtentRepair { repair_id: ReconciliationId(142), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.857 INFO [1] client ExtentRepair { repair_id: ReconciliationId(142), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.857 INFO [1] No action required ReconciliationId(142)
Sep 22 23:22:16.857 INFO [2] received reconcile message
Sep 22 23:22:16.857 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(142), op: ExtentRepair { repair_id: ReconciliationId(142), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.857 INFO [2] client ExtentRepair { repair_id: ReconciliationId(142), extent_id: 78, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.857 INFO [2] No action required ReconciliationId(142)
Sep 22 23:22:16.857 DEBG 142 Repair extent 78 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.857 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/04E.copy"
Sep 22 23:22:16.920 INFO accepted connection, remote_addr: 127.0.0.1:34914, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.920 TRCE incoming request, uri: /extent/78/files, method: GET, req_id: 7fb138ee-9655-472d-9c2b-993ae6a1816b, remote_addr: 127.0.0.1:34914, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.920 INFO request completed, latency_us: 201, response_code: 200, uri: /extent/78/files, method: GET, req_id: 7fb138ee-9655-472d-9c2b-993ae6a1816b, remote_addr: 127.0.0.1:34914, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.920 INFO eid:78 Found repair files: ["04E", "04E.db"]
Sep 22 23:22:16.921 TRCE incoming request, uri: /newextent/78/data, method: GET, req_id: 5b404dd1-9460-4baf-a26b-15fe5306f7ab, remote_addr: 127.0.0.1:34914, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.921 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/78/data, method: GET, req_id: 5b404dd1-9460-4baf-a26b-15fe5306f7ab, remote_addr: 127.0.0.1:34914, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.922 DEBG [rc] retire 1074 clears [JobId(1073), JobId(1074)], : downstairs
Sep 22 23:22:16.926 TRCE incoming request, uri: /newextent/78/db, method: GET, req_id: 87bb67ce-8c37-42b2-b323-377daad1fb95, remote_addr: 127.0.0.1:34914, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.926 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/78/db, method: GET, req_id: 87bb67ce-8c37-42b2-b323-377daad1fb95, remote_addr: 127.0.0.1:34914, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.927 DEBG up_ds_listen was notified
Sep 22 23:22:16.927 DEBG up_ds_listen process 1076
Sep 22 23:22:16.927 DEBG [A] ack job 1076:77, : downstairs
Sep 22 23:22:16.927 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:16.927 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/04E.copy" to "/tmp/downstairs-zrMnlo6G/00/000/04E.replace"
Sep 22 23:22:16.927 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.928 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/04E.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.929 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04E"
Sep 22 23:22:16.929 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04E.db"
Sep 22 23:22:16.929 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.929 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/04E.replace" to "/tmp/downstairs-zrMnlo6G/00/000/04E.completed"
Sep 22 23:22:16.929 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.929 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:16.929 DEBG [0] It's time to notify for 142
Sep 22 23:22:16.929 INFO Completion from [0] id:142 status:true
Sep 22 23:22:16.929 INFO [143/752] Repair commands completed
Sep 22 23:22:16.929 INFO Pop front: ReconcileIO { id: ReconciliationId(143), op: ExtentReopen { repair_id: ReconciliationId(143), extent_id: 78 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.929 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.929 INFO [0] received reconcile message
Sep 22 23:22:16.929 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(143), op: ExtentReopen { repair_id: ReconciliationId(143), extent_id: 78 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.929 INFO [0] client ExtentReopen { repair_id: ReconciliationId(143), extent_id: 78 }
Sep 22 23:22:16.929 INFO [1] received reconcile message
Sep 22 23:22:16.929 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(143), op: ExtentReopen { repair_id: ReconciliationId(143), extent_id: 78 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.929 INFO [1] client ExtentReopen { repair_id: ReconciliationId(143), extent_id: 78 }
Sep 22 23:22:16.929 INFO [2] received reconcile message
Sep 22 23:22:16.929 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(143), op: ExtentReopen { repair_id: ReconciliationId(143), extent_id: 78 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.929 INFO [2] client ExtentReopen { repair_id: ReconciliationId(143), extent_id: 78 }
Sep 22 23:22:16.930 DEBG 143 Reopen extent 78
Sep 22 23:22:16.930 DEBG 143 Reopen extent 78
Sep 22 23:22:16.931 DEBG 143 Reopen extent 78
Sep 22 23:22:16.931 DEBG [2] It's time to notify for 143
Sep 22 23:22:16.931 INFO Completion from [2] id:143 status:true
Sep 22 23:22:16.931 INFO [144/752] Repair commands completed
Sep 22 23:22:16.931 INFO Pop front: ReconcileIO { id: ReconciliationId(144), op: ExtentFlush { repair_id: ReconciliationId(144), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.931 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.931 INFO [0] received reconcile message
Sep 22 23:22:16.932 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(144), op: ExtentFlush { repair_id: ReconciliationId(144), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.932 INFO [0] client ExtentFlush { repair_id: ReconciliationId(144), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.932 INFO [1] received reconcile message
Sep 22 23:22:16.932 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(144), op: ExtentFlush { repair_id: ReconciliationId(144), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:16.932 INFO [1] client ExtentFlush { repair_id: ReconciliationId(144), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.932 INFO [2] received reconcile message
Sep 22 23:22:16.932 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(144), op: ExtentFlush { repair_id: ReconciliationId(144), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.932 INFO [2] client ExtentFlush { repair_id: ReconciliationId(144), extent_id: 52, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:16.932 DEBG 144 Flush extent 52 with f:2 g:2
Sep 22 23:22:16.932 DEBG Flush just extent 52 with f:2 and g:2
Sep 22 23:22:16.932 DEBG [1] It's time to notify for 144
Sep 22 23:22:16.932 INFO Completion from [1] id:144 status:true
Sep 22 23:22:16.932 INFO [145/752] Repair commands completed
Sep 22 23:22:16.932 INFO Pop front: ReconcileIO { id: ReconciliationId(145), op: ExtentClose { repair_id: ReconciliationId(145), extent_id: 52 }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.932 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.932 INFO [0] received reconcile message
Sep 22 23:22:16.932 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(145), op: ExtentClose { repair_id: ReconciliationId(145), extent_id: 52 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.932 INFO [0] client ExtentClose { repair_id: ReconciliationId(145), extent_id: 52 }
Sep 22 23:22:16.932 INFO [1] received reconcile message
Sep 22 23:22:16.932 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(145), op: ExtentClose { repair_id: ReconciliationId(145), extent_id: 52 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.932 INFO [1] client ExtentClose { repair_id: ReconciliationId(145), extent_id: 52 }
Sep 22 23:22:16.932 INFO [2] received reconcile message
Sep 22 23:22:16.932 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(145), op: ExtentClose { repair_id: ReconciliationId(145), extent_id: 52 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:16.932 INFO [2] client ExtentClose { repair_id: ReconciliationId(145), extent_id: 52 }
Sep 22 23:22:16.932 DEBG 145 Close extent 52
Sep 22 23:22:16.933 DEBG 145 Close extent 52
Sep 22 23:22:16.933 DEBG 145 Close extent 52
Sep 22 23:22:16.933 DEBG [2] It's time to notify for 145
Sep 22 23:22:16.933 INFO Completion from [2] id:145 status:true
Sep 22 23:22:16.933 INFO [146/752] Repair commands completed
Sep 22 23:22:16.933 INFO Pop front: ReconcileIO { id: ReconciliationId(146), op: ExtentRepair { repair_id: ReconciliationId(146), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:16.933 INFO Sent repair work, now wait for resp
Sep 22 23:22:16.934 INFO [0] received reconcile message
Sep 22 23:22:16.934 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(146), op: ExtentRepair { repair_id: ReconciliationId(146), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:16.934 INFO [0] client ExtentRepair { repair_id: ReconciliationId(146), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.934 INFO [0] Sending repair request ReconciliationId(146)
Sep 22 23:22:16.934 INFO [1] received reconcile message
Sep 22 23:22:16.934 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(146), op: ExtentRepair { repair_id: ReconciliationId(146), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:16.934 INFO [1] client ExtentRepair { repair_id: ReconciliationId(146), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.934 INFO [1] No action required ReconciliationId(146)
Sep 22 23:22:16.934 INFO [2] received reconcile message
Sep 22 23:22:16.934 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(146), op: ExtentRepair { repair_id: ReconciliationId(146), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:16.934 INFO [2] client ExtentRepair { repair_id: ReconciliationId(146), extent_id: 52, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:16.934 INFO [2] No action required ReconciliationId(146)
Sep 22 23:22:16.934 DEBG 146 Repair extent 52 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:16.934 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/034.copy"
Sep 22 23:22:16.934 DEBG IO Flush 1078 has deps [JobId(1077), JobId(1076)]
Sep 22 23:22:16.937 INFO [lossy] skipping 1076
Sep 22 23:22:16.937 DEBG Flush     :1076 extent_limit None deps:[JobId(1075), JobId(1074)] res:true f:27 g:1
Sep 22 23:22:16.943 DEBG Read      :1077 deps:[JobId(1076)] res:true
Sep 22 23:22:16.998 INFO accepted connection, remote_addr: 127.0.0.1:52058, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.998 TRCE incoming request, uri: /extent/52/files, method: GET, req_id: 363ed5f9-5739-4d19-ac30-c7a188813cab, remote_addr: 127.0.0.1:52058, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.999 INFO request completed, latency_us: 255, response_code: 200, uri: /extent/52/files, method: GET, req_id: 363ed5f9-5739-4d19-ac30-c7a188813cab, remote_addr: 127.0.0.1:52058, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:16.999 INFO eid:52 Found repair files: ["034", "034.db"]
Sep 22 23:22:16.999 TRCE incoming request, uri: /newextent/52/data, method: GET, req_id: 3c3825dd-412b-45c6-a583-45439085c71e, remote_addr: 127.0.0.1:52058, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.000 INFO request completed, latency_us: 361, response_code: 200, uri: /newextent/52/data, method: GET, req_id: 3c3825dd-412b-45c6-a583-45439085c71e, remote_addr: 127.0.0.1:52058, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.005 TRCE incoming request, uri: /newextent/52/db, method: GET, req_id: a87bbae6-48c7-4d09-a0e2-a8f96e5f4bb5, remote_addr: 127.0.0.1:52058, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.005 INFO request completed, latency_us: 317, response_code: 200, uri: /newextent/52/db, method: GET, req_id: a87bbae6-48c7-4d09-a0e2-a8f96e5f4bb5, remote_addr: 127.0.0.1:52058, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.007 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/034.copy" to "/tmp/downstairs-zrMnlo6G/00/000/034.replace"
Sep 22 23:22:17.007 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.008 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/034.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.008 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/034"
Sep 22 23:22:17.008 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/034.db"
Sep 22 23:22:17.008 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.008 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/034.replace" to "/tmp/downstairs-zrMnlo6G/00/000/034.completed"
Sep 22 23:22:17.008 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.008 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.009 DEBG [0] It's time to notify for 146
Sep 22 23:22:17.009 INFO Completion from [0] id:146 status:true
Sep 22 23:22:17.009 INFO [147/752] Repair commands completed
Sep 22 23:22:17.009 INFO Pop front: ReconcileIO { id: ReconciliationId(147), op: ExtentReopen { repair_id: ReconciliationId(147), extent_id: 52 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.009 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.009 INFO [0] received reconcile message
Sep 22 23:22:17.009 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(147), op: ExtentReopen { repair_id: ReconciliationId(147), extent_id: 52 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.009 INFO [0] client ExtentReopen { repair_id: ReconciliationId(147), extent_id: 52 }
Sep 22 23:22:17.009 INFO [1] received reconcile message
Sep 22 23:22:17.009 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(147), op: ExtentReopen { repair_id: ReconciliationId(147), extent_id: 52 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.009 INFO [1] client ExtentReopen { repair_id: ReconciliationId(147), extent_id: 52 }
Sep 22 23:22:17.009 INFO [2] received reconcile message
Sep 22 23:22:17.009 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(147), op: ExtentReopen { repair_id: ReconciliationId(147), extent_id: 52 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.009 INFO [2] client ExtentReopen { repair_id: ReconciliationId(147), extent_id: 52 }
Sep 22 23:22:17.009 DEBG 147 Reopen extent 52
Sep 22 23:22:17.010 DEBG 147 Reopen extent 52
Sep 22 23:22:17.011 DEBG 147 Reopen extent 52
Sep 22 23:22:17.011 DEBG [2] It's time to notify for 147
Sep 22 23:22:17.011 INFO Completion from [2] id:147 status:true
Sep 22 23:22:17.011 INFO [148/752] Repair commands completed
Sep 22 23:22:17.011 INFO Pop front: ReconcileIO { id: ReconciliationId(148), op: ExtentFlush { repair_id: ReconciliationId(148), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.011 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.011 INFO [0] received reconcile message
Sep 22 23:22:17.011 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(148), op: ExtentFlush { repair_id: ReconciliationId(148), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.011 INFO [0] client ExtentFlush { repair_id: ReconciliationId(148), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.011 INFO [1] received reconcile message
Sep 22 23:22:17.011 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(148), op: ExtentFlush { repair_id: ReconciliationId(148), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.011 INFO [1] client ExtentFlush { repair_id: ReconciliationId(148), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.012 INFO [2] received reconcile message
Sep 22 23:22:17.012 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(148), op: ExtentFlush { repair_id: ReconciliationId(148), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.012 INFO [2] client ExtentFlush { repair_id: ReconciliationId(148), extent_id: 172, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.012 DEBG 148 Flush extent 172 with f:2 g:2
Sep 22 23:22:17.012 DEBG Flush just extent 172 with f:2 and g:2
Sep 22 23:22:17.012 DEBG [1] It's time to notify for 148
Sep 22 23:22:17.012 INFO Completion from [1] id:148 status:true
Sep 22 23:22:17.012 INFO [149/752] Repair commands completed
Sep 22 23:22:17.012 INFO Pop front: ReconcileIO { id: ReconciliationId(149), op: ExtentClose { repair_id: ReconciliationId(149), extent_id: 172 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.012 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.012 INFO [0] received reconcile message
Sep 22 23:22:17.012 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(149), op: ExtentClose { repair_id: ReconciliationId(149), extent_id: 172 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.012 INFO [0] client ExtentClose { repair_id: ReconciliationId(149), extent_id: 172 }
Sep 22 23:22:17.012 INFO [1] received reconcile message
Sep 22 23:22:17.012 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(149), op: ExtentClose { repair_id: ReconciliationId(149), extent_id: 172 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.012 INFO [1] client ExtentClose { repair_id: ReconciliationId(149), extent_id: 172 }
Sep 22 23:22:17.012 INFO [2] received reconcile message
Sep 22 23:22:17.012 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(149), op: ExtentClose { repair_id: ReconciliationId(149), extent_id: 172 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.012 INFO [2] client ExtentClose { repair_id: ReconciliationId(149), extent_id: 172 }
Sep 22 23:22:17.012 DEBG 149 Close extent 172
Sep 22 23:22:17.013 DEBG 149 Close extent 172
Sep 22 23:22:17.013 DEBG 149 Close extent 172
Sep 22 23:22:17.013 DEBG [2] It's time to notify for 149
Sep 22 23:22:17.013 INFO Completion from [2] id:149 status:true
Sep 22 23:22:17.013 INFO [150/752] Repair commands completed
Sep 22 23:22:17.013 INFO Pop front: ReconcileIO { id: ReconciliationId(150), op: ExtentRepair { repair_id: ReconciliationId(150), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.013 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.013 INFO [0] received reconcile message
Sep 22 23:22:17.013 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(150), op: ExtentRepair { repair_id: ReconciliationId(150), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.013 INFO [0] client ExtentRepair { repair_id: ReconciliationId(150), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.013 INFO [0] Sending repair request ReconciliationId(150)
Sep 22 23:22:17.014 INFO [1] received reconcile message
Sep 22 23:22:17.014 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(150), op: ExtentRepair { repair_id: ReconciliationId(150), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.014 INFO [1] client ExtentRepair { repair_id: ReconciliationId(150), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.014 INFO [1] No action required ReconciliationId(150)
Sep 22 23:22:17.014 INFO [2] received reconcile message
Sep 22 23:22:17.014 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(150), op: ExtentRepair { repair_id: ReconciliationId(150), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.014 INFO [2] client ExtentRepair { repair_id: ReconciliationId(150), extent_id: 172, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.014 INFO [2] No action required ReconciliationId(150)
Sep 22 23:22:17.014 DEBG 150 Repair extent 172 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.014 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0AC.copy"
Sep 22 23:22:17.078 INFO accepted connection, remote_addr: 127.0.0.1:62113, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.078 TRCE incoming request, uri: /extent/172/files, method: GET, req_id: a38ba9a9-76dc-4fb1-bf59-413d19ce07d3, remote_addr: 127.0.0.1:62113, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.079 INFO request completed, latency_us: 210, response_code: 200, uri: /extent/172/files, method: GET, req_id: a38ba9a9-76dc-4fb1-bf59-413d19ce07d3, remote_addr: 127.0.0.1:62113, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.079 INFO eid:172 Found repair files: ["0AC", "0AC.db"]
Sep 22 23:22:17.079 TRCE incoming request, uri: /newextent/172/data, method: GET, req_id: 2e306838-2ae4-408d-98e5-75df94fc25ed, remote_addr: 127.0.0.1:62113, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.080 INFO request completed, latency_us: 325, response_code: 200, uri: /newextent/172/data, method: GET, req_id: 2e306838-2ae4-408d-98e5-75df94fc25ed, remote_addr: 127.0.0.1:62113, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.084 TRCE incoming request, uri: /newextent/172/db, method: GET, req_id: a6273c2d-8391-4583-b3a6-99403a1c8d7a, remote_addr: 127.0.0.1:62113, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.085 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/172/db, method: GET, req_id: a6273c2d-8391-4583-b3a6-99403a1c8d7a, remote_addr: 127.0.0.1:62113, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.086 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0AC.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0AC.replace"
Sep 22 23:22:17.086 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.086 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0AC.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.087 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AC"
Sep 22 23:22:17.087 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AC.db"
Sep 22 23:22:17.087 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.087 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0AC.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0AC.completed"
Sep 22 23:22:17.087 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.087 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.087 DEBG [0] It's time to notify for 150
Sep 22 23:22:17.087 INFO Completion from [0] id:150 status:true
Sep 22 23:22:17.087 INFO [151/752] Repair commands completed
Sep 22 23:22:17.087 INFO Pop front: ReconcileIO { id: ReconciliationId(151), op: ExtentReopen { repair_id: ReconciliationId(151), extent_id: 172 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.087 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.087 INFO [0] received reconcile message
Sep 22 23:22:17.087 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(151), op: ExtentReopen { repair_id: ReconciliationId(151), extent_id: 172 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.087 INFO [0] client ExtentReopen { repair_id: ReconciliationId(151), extent_id: 172 }
Sep 22 23:22:17.087 INFO [1] received reconcile message
Sep 22 23:22:17.087 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(151), op: ExtentReopen { repair_id: ReconciliationId(151), extent_id: 172 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.088 INFO [1] client ExtentReopen { repair_id: ReconciliationId(151), extent_id: 172 }
Sep 22 23:22:17.088 INFO [2] received reconcile message
Sep 22 23:22:17.088 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(151), op: ExtentReopen { repair_id: ReconciliationId(151), extent_id: 172 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.088 INFO [2] client ExtentReopen { repair_id: ReconciliationId(151), extent_id: 172 }
Sep 22 23:22:17.088 DEBG 151 Reopen extent 172
Sep 22 23:22:17.088 DEBG 151 Reopen extent 172
Sep 22 23:22:17.089 DEBG 151 Reopen extent 172
Sep 22 23:22:17.089 DEBG [2] It's time to notify for 151
Sep 22 23:22:17.089 INFO Completion from [2] id:151 status:true
Sep 22 23:22:17.089 INFO [152/752] Repair commands completed
Sep 22 23:22:17.089 INFO Pop front: ReconcileIO { id: ReconciliationId(152), op: ExtentFlush { repair_id: ReconciliationId(152), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.090 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.090 INFO [0] received reconcile message
Sep 22 23:22:17.090 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(152), op: ExtentFlush { repair_id: ReconciliationId(152), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.090 INFO [0] client ExtentFlush { repair_id: ReconciliationId(152), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.090 INFO [1] received reconcile message
Sep 22 23:22:17.090 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(152), op: ExtentFlush { repair_id: ReconciliationId(152), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.090 INFO [1] client ExtentFlush { repair_id: ReconciliationId(152), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.090 INFO [2] received reconcile message
Sep 22 23:22:17.090 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(152), op: ExtentFlush { repair_id: ReconciliationId(152), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.090 INFO [2] client ExtentFlush { repair_id: ReconciliationId(152), extent_id: 183, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.090 DEBG 152 Flush extent 183 with f:2 g:2
Sep 22 23:22:17.090 DEBG Flush just extent 183 with f:2 and g:2
Sep 22 23:22:17.090 DEBG [1] It's time to notify for 152
Sep 22 23:22:17.090 INFO Completion from [1] id:152 status:true
Sep 22 23:22:17.090 INFO [153/752] Repair commands completed
Sep 22 23:22:17.090 INFO Pop front: ReconcileIO { id: ReconciliationId(153), op: ExtentClose { repair_id: ReconciliationId(153), extent_id: 183 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.090 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.090 INFO [0] received reconcile message
Sep 22 23:22:17.090 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(153), op: ExtentClose { repair_id: ReconciliationId(153), extent_id: 183 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.090 INFO [0] client ExtentClose { repair_id: ReconciliationId(153), extent_id: 183 }
Sep 22 23:22:17.090 INFO [1] received reconcile message
Sep 22 23:22:17.090 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(153), op: ExtentClose { repair_id: ReconciliationId(153), extent_id: 183 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.090 INFO [1] client ExtentClose { repair_id: ReconciliationId(153), extent_id: 183 }
Sep 22 23:22:17.090 INFO [2] received reconcile message
Sep 22 23:22:17.090 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(153), op: ExtentClose { repair_id: ReconciliationId(153), extent_id: 183 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.090 INFO [2] client ExtentClose { repair_id: ReconciliationId(153), extent_id: 183 }
Sep 22 23:22:17.090 DEBG 153 Close extent 183
Sep 22 23:22:17.091 DEBG 153 Close extent 183
Sep 22 23:22:17.091 DEBG 153 Close extent 183
Sep 22 23:22:17.091 DEBG [2] It's time to notify for 153
Sep 22 23:22:17.091 INFO Completion from [2] id:153 status:true
Sep 22 23:22:17.091 INFO [154/752] Repair commands completed
Sep 22 23:22:17.092 INFO Pop front: ReconcileIO { id: ReconciliationId(154), op: ExtentRepair { repair_id: ReconciliationId(154), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.092 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.092 INFO [0] received reconcile message
Sep 22 23:22:17.092 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(154), op: ExtentRepair { repair_id: ReconciliationId(154), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.092 INFO [0] client ExtentRepair { repair_id: ReconciliationId(154), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.092 INFO [0] Sending repair request ReconciliationId(154)
Sep 22 23:22:17.092 INFO [1] received reconcile message
Sep 22 23:22:17.092 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(154), op: ExtentRepair { repair_id: ReconciliationId(154), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.092 INFO [1] client ExtentRepair { repair_id: ReconciliationId(154), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.092 INFO [1] No action required ReconciliationId(154)
Sep 22 23:22:17.092 INFO [2] received reconcile message
Sep 22 23:22:17.092 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(154), op: ExtentRepair { repair_id: ReconciliationId(154), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.092 INFO [2] client ExtentRepair { repair_id: ReconciliationId(154), extent_id: 183, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.092 INFO [2] No action required ReconciliationId(154)
Sep 22 23:22:17.092 DEBG 154 Repair extent 183 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.092 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0B7.copy"
Sep 22 23:22:17.156 INFO accepted connection, remote_addr: 127.0.0.1:39922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.156 TRCE incoming request, uri: /extent/183/files, method: GET, req_id: d4a4981d-c65e-4879-beff-a128439ccd9d, remote_addr: 127.0.0.1:39922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.156 INFO request completed, latency_us: 203, response_code: 200, uri: /extent/183/files, method: GET, req_id: d4a4981d-c65e-4879-beff-a128439ccd9d, remote_addr: 127.0.0.1:39922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.157 INFO eid:183 Found repair files: ["0B7", "0B7.db"]
Sep 22 23:22:17.157 TRCE incoming request, uri: /newextent/183/data, method: GET, req_id: 8979f112-da5d-46c8-9341-3c3b269d12dd, remote_addr: 127.0.0.1:39922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.157 INFO request completed, latency_us: 325, response_code: 200, uri: /newextent/183/data, method: GET, req_id: 8979f112-da5d-46c8-9341-3c3b269d12dd, remote_addr: 127.0.0.1:39922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.162 TRCE incoming request, uri: /newextent/183/db, method: GET, req_id: 1c39fc90-df86-4d84-9f18-61a7b9923cfa, remote_addr: 127.0.0.1:39922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.162 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/183/db, method: GET, req_id: 1c39fc90-df86-4d84-9f18-61a7b9923cfa, remote_addr: 127.0.0.1:39922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.163 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0B7.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0B7.replace"
Sep 22 23:22:17.163 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.164 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0B7.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.164 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B7"
Sep 22 23:22:17.165 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B7.db"
Sep 22 23:22:17.165 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.165 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0B7.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0B7.completed"
Sep 22 23:22:17.165 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.165 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.165 DEBG [0] It's time to notify for 154
Sep 22 23:22:17.165 INFO Completion from [0] id:154 status:true
Sep 22 23:22:17.165 INFO [155/752] Repair commands completed
Sep 22 23:22:17.165 INFO Pop front: ReconcileIO { id: ReconciliationId(155), op: ExtentReopen { repair_id: ReconciliationId(155), extent_id: 183 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.165 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.165 INFO [0] received reconcile message
Sep 22 23:22:17.165 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(155), op: ExtentReopen { repair_id: ReconciliationId(155), extent_id: 183 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.165 INFO [0] client ExtentReopen { repair_id: ReconciliationId(155), extent_id: 183 }
Sep 22 23:22:17.165 INFO [1] received reconcile message
Sep 22 23:22:17.165 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(155), op: ExtentReopen { repair_id: ReconciliationId(155), extent_id: 183 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.165 INFO [1] client ExtentReopen { repair_id: ReconciliationId(155), extent_id: 183 }
Sep 22 23:22:17.165 INFO [2] received reconcile message
Sep 22 23:22:17.165 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(155), op: ExtentReopen { repair_id: ReconciliationId(155), extent_id: 183 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.165 INFO [2] client ExtentReopen { repair_id: ReconciliationId(155), extent_id: 183 }
Sep 22 23:22:17.165 DEBG 155 Reopen extent 183
Sep 22 23:22:17.166 DEBG 155 Reopen extent 183
Sep 22 23:22:17.166 DEBG 155 Reopen extent 183
Sep 22 23:22:17.167 DEBG [2] It's time to notify for 155
Sep 22 23:22:17.167 INFO Completion from [2] id:155 status:true
Sep 22 23:22:17.167 INFO [156/752] Repair commands completed
Sep 22 23:22:17.167 INFO Pop front: ReconcileIO { id: ReconciliationId(156), op: ExtentFlush { repair_id: ReconciliationId(156), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.167 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.167 INFO [0] received reconcile message
Sep 22 23:22:17.167 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(156), op: ExtentFlush { repair_id: ReconciliationId(156), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.167 INFO [0] client ExtentFlush { repair_id: ReconciliationId(156), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.167 INFO [1] received reconcile message
Sep 22 23:22:17.167 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(156), op: ExtentFlush { repair_id: ReconciliationId(156), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.167 INFO [1] client ExtentFlush { repair_id: ReconciliationId(156), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.167 INFO [2] received reconcile message
Sep 22 23:22:17.167 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(156), op: ExtentFlush { repair_id: ReconciliationId(156), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.167 INFO [2] client ExtentFlush { repair_id: ReconciliationId(156), extent_id: 11, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.168 DEBG 156 Flush extent 11 with f:2 g:2
Sep 22 23:22:17.168 DEBG Flush just extent 11 with f:2 and g:2
Sep 22 23:22:17.168 DEBG [1] It's time to notify for 156
Sep 22 23:22:17.168 INFO Completion from [1] id:156 status:true
Sep 22 23:22:17.168 INFO [157/752] Repair commands completed
Sep 22 23:22:17.168 INFO Pop front: ReconcileIO { id: ReconciliationId(157), op: ExtentClose { repair_id: ReconciliationId(157), extent_id: 11 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.168 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.168 INFO [0] received reconcile message
Sep 22 23:22:17.168 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(157), op: ExtentClose { repair_id: ReconciliationId(157), extent_id: 11 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.168 INFO [0] client ExtentClose { repair_id: ReconciliationId(157), extent_id: 11 }
Sep 22 23:22:17.168 INFO [1] received reconcile message
Sep 22 23:22:17.168 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(157), op: ExtentClose { repair_id: ReconciliationId(157), extent_id: 11 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.168 INFO [1] client ExtentClose { repair_id: ReconciliationId(157), extent_id: 11 }
Sep 22 23:22:17.168 INFO [2] received reconcile message
Sep 22 23:22:17.168 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(157), op: ExtentClose { repair_id: ReconciliationId(157), extent_id: 11 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.168 INFO [2] client ExtentClose { repair_id: ReconciliationId(157), extent_id: 11 }
Sep 22 23:22:17.168 DEBG 157 Close extent 11
Sep 22 23:22:17.168 DEBG 157 Close extent 11
Sep 22 23:22:17.169 DEBG 157 Close extent 11
Sep 22 23:22:17.169 DEBG [2] It's time to notify for 157
Sep 22 23:22:17.169 INFO Completion from [2] id:157 status:true
Sep 22 23:22:17.169 INFO [158/752] Repair commands completed
Sep 22 23:22:17.169 INFO Pop front: ReconcileIO { id: ReconciliationId(158), op: ExtentRepair { repair_id: ReconciliationId(158), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.169 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.169 INFO [0] received reconcile message
Sep 22 23:22:17.169 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(158), op: ExtentRepair { repair_id: ReconciliationId(158), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.169 INFO [0] client ExtentRepair { repair_id: ReconciliationId(158), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.169 INFO [0] Sending repair request ReconciliationId(158)
Sep 22 23:22:17.169 INFO [1] received reconcile message
Sep 22 23:22:17.169 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(158), op: ExtentRepair { repair_id: ReconciliationId(158), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.169 INFO [1] client ExtentRepair { repair_id: ReconciliationId(158), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.169 INFO [1] No action required ReconciliationId(158)
Sep 22 23:22:17.169 INFO [2] received reconcile message
Sep 22 23:22:17.169 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(158), op: ExtentRepair { repair_id: ReconciliationId(158), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.169 INFO [2] client ExtentRepair { repair_id: ReconciliationId(158), extent_id: 11, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.170 INFO [2] No action required ReconciliationId(158)
Sep 22 23:22:17.170 DEBG 158 Repair extent 11 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.170 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/00B.copy"
Sep 22 23:22:17.231 INFO accepted connection, remote_addr: 127.0.0.1:47163, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.231 TRCE incoming request, uri: /extent/11/files, method: GET, req_id: 38dc2dba-2848-4846-9b18-a27b7fdc59d6, remote_addr: 127.0.0.1:47163, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.232 INFO request completed, latency_us: 203, response_code: 200, uri: /extent/11/files, method: GET, req_id: 38dc2dba-2848-4846-9b18-a27b7fdc59d6, remote_addr: 127.0.0.1:47163, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.232 INFO eid:11 Found repair files: ["00B", "00B.db"]
Sep 22 23:22:17.232 TRCE incoming request, uri: /newextent/11/data, method: GET, req_id: 9c870808-9285-4281-9866-eb571d7336b3, remote_addr: 127.0.0.1:47163, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.233 INFO request completed, latency_us: 319, response_code: 200, uri: /newextent/11/data, method: GET, req_id: 9c870808-9285-4281-9866-eb571d7336b3, remote_addr: 127.0.0.1:47163, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.237 TRCE incoming request, uri: /newextent/11/db, method: GET, req_id: 5a5f4c22-37de-4257-84f1-25a808e776dd, remote_addr: 127.0.0.1:47163, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.237 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/11/db, method: GET, req_id: 5a5f4c22-37de-4257-84f1-25a808e776dd, remote_addr: 127.0.0.1:47163, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.239 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/00B.copy" to "/tmp/downstairs-zrMnlo6G/00/000/00B.replace"
Sep 22 23:22:17.239 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.239 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/00B.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.240 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00B"
Sep 22 23:22:17.240 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00B.db"
Sep 22 23:22:17.240 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.240 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/00B.replace" to "/tmp/downstairs-zrMnlo6G/00/000/00B.completed"
Sep 22 23:22:17.240 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.240 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.240 DEBG [0] It's time to notify for 158
Sep 22 23:22:17.240 INFO Completion from [0] id:158 status:true
Sep 22 23:22:17.240 INFO [159/752] Repair commands completed
Sep 22 23:22:17.240 INFO Pop front: ReconcileIO { id: ReconciliationId(159), op: ExtentReopen { repair_id: ReconciliationId(159), extent_id: 11 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.240 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.240 INFO [0] received reconcile message
Sep 22 23:22:17.240 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(159), op: ExtentReopen { repair_id: ReconciliationId(159), extent_id: 11 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.240 INFO [0] client ExtentReopen { repair_id: ReconciliationId(159), extent_id: 11 }
Sep 22 23:22:17.240 INFO [1] received reconcile message
Sep 22 23:22:17.240 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(159), op: ExtentReopen { repair_id: ReconciliationId(159), extent_id: 11 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.240 INFO [1] client ExtentReopen { repair_id: ReconciliationId(159), extent_id: 11 }
Sep 22 23:22:17.241 INFO [2] received reconcile message
Sep 22 23:22:17.241 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(159), op: ExtentReopen { repair_id: ReconciliationId(159), extent_id: 11 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.241 INFO [2] client ExtentReopen { repair_id: ReconciliationId(159), extent_id: 11 }
Sep 22 23:22:17.241 DEBG 159 Reopen extent 11
Sep 22 23:22:17.241 DEBG 159 Reopen extent 11
Sep 22 23:22:17.242 DEBG 159 Reopen extent 11
Sep 22 23:22:17.242 DEBG [2] It's time to notify for 159
Sep 22 23:22:17.242 INFO Completion from [2] id:159 status:true
Sep 22 23:22:17.242 INFO [160/752] Repair commands completed
Sep 22 23:22:17.242 INFO Pop front: ReconcileIO { id: ReconciliationId(160), op: ExtentFlush { repair_id: ReconciliationId(160), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.243 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.243 INFO [0] received reconcile message
Sep 22 23:22:17.243 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(160), op: ExtentFlush { repair_id: ReconciliationId(160), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.243 INFO [0] client ExtentFlush { repair_id: ReconciliationId(160), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.243 INFO [1] received reconcile message
Sep 22 23:22:17.243 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(160), op: ExtentFlush { repair_id: ReconciliationId(160), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.243 INFO [1] client ExtentFlush { repair_id: ReconciliationId(160), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.243 INFO [2] received reconcile message
Sep 22 23:22:17.243 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(160), op: ExtentFlush { repair_id: ReconciliationId(160), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.243 INFO [2] client ExtentFlush { repair_id: ReconciliationId(160), extent_id: 184, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.243 DEBG 160 Flush extent 184 with f:2 g:2
Sep 22 23:22:17.243 DEBG Flush just extent 184 with f:2 and g:2
Sep 22 23:22:17.243 DEBG [1] It's time to notify for 160
Sep 22 23:22:17.243 INFO Completion from [1] id:160 status:true
Sep 22 23:22:17.243 INFO [161/752] Repair commands completed
Sep 22 23:22:17.243 INFO Pop front: ReconcileIO { id: ReconciliationId(161), op: ExtentClose { repair_id: ReconciliationId(161), extent_id: 184 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.243 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.243 INFO [0] received reconcile message
Sep 22 23:22:17.243 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(161), op: ExtentClose { repair_id: ReconciliationId(161), extent_id: 184 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.243 INFO [0] client ExtentClose { repair_id: ReconciliationId(161), extent_id: 184 }
Sep 22 23:22:17.243 INFO [1] received reconcile message
Sep 22 23:22:17.243 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(161), op: ExtentClose { repair_id: ReconciliationId(161), extent_id: 184 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.243 INFO [1] client ExtentClose { repair_id: ReconciliationId(161), extent_id: 184 }
Sep 22 23:22:17.243 INFO [2] received reconcile message
Sep 22 23:22:17.243 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(161), op: ExtentClose { repair_id: ReconciliationId(161), extent_id: 184 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.243 INFO [2] client ExtentClose { repair_id: ReconciliationId(161), extent_id: 184 }
Sep 22 23:22:17.243 DEBG 161 Close extent 184
Sep 22 23:22:17.244 DEBG 161 Close extent 184
Sep 22 23:22:17.244 DEBG 161 Close extent 184
Sep 22 23:22:17.244 DEBG [2] It's time to notify for 161
Sep 22 23:22:17.244 INFO Completion from [2] id:161 status:true
Sep 22 23:22:17.244 INFO [162/752] Repair commands completed
Sep 22 23:22:17.244 INFO Pop front: ReconcileIO { id: ReconciliationId(162), op: ExtentRepair { repair_id: ReconciliationId(162), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.244 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.245 INFO [0] received reconcile message
Sep 22 23:22:17.245 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(162), op: ExtentRepair { repair_id: ReconciliationId(162), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.245 INFO [0] client ExtentRepair { repair_id: ReconciliationId(162), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.245 INFO [0] Sending repair request ReconciliationId(162)
Sep 22 23:22:17.245 INFO [1] received reconcile message
Sep 22 23:22:17.245 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(162), op: ExtentRepair { repair_id: ReconciliationId(162), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.245 INFO [1] client ExtentRepair { repair_id: ReconciliationId(162), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.245 INFO [1] No action required ReconciliationId(162)
Sep 22 23:22:17.245 INFO [2] received reconcile message
Sep 22 23:22:17.245 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(162), op: ExtentRepair { repair_id: ReconciliationId(162), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.245 INFO [2] client ExtentRepair { repair_id: ReconciliationId(162), extent_id: 184, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.245 INFO [2] No action required ReconciliationId(162)
Sep 22 23:22:17.245 DEBG 162 Repair extent 184 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.245 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0B8.copy"
Sep 22 23:22:17.309 INFO accepted connection, remote_addr: 127.0.0.1:55201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.309 TRCE incoming request, uri: /extent/184/files, method: GET, req_id: 386d5a37-580e-47bd-9e21-a1f4974905fc, remote_addr: 127.0.0.1:55201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.310 INFO request completed, latency_us: 196, response_code: 200, uri: /extent/184/files, method: GET, req_id: 386d5a37-580e-47bd-9e21-a1f4974905fc, remote_addr: 127.0.0.1:55201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.310 INFO eid:184 Found repair files: ["0B8", "0B8.db"]
Sep 22 23:22:17.310 TRCE incoming request, uri: /newextent/184/data, method: GET, req_id: 17616a71-7f55-46d4-bdc6-de4f76590f02, remote_addr: 127.0.0.1:55201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.310 INFO request completed, latency_us: 316, response_code: 200, uri: /newextent/184/data, method: GET, req_id: 17616a71-7f55-46d4-bdc6-de4f76590f02, remote_addr: 127.0.0.1:55201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.315 TRCE incoming request, uri: /newextent/184/db, method: GET, req_id: 7dc18af3-7c2c-40ab-877b-3cf08acc357b, remote_addr: 127.0.0.1:55201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.315 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/184/db, method: GET, req_id: 7dc18af3-7c2c-40ab-877b-3cf08acc357b, remote_addr: 127.0.0.1:55201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.317 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0B8.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0B8.replace"
Sep 22 23:22:17.317 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.317 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0B8.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.318 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B8"
Sep 22 23:22:17.318 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B8.db"
Sep 22 23:22:17.318 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.318 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0B8.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0B8.completed"
Sep 22 23:22:17.318 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.318 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.318 DEBG [0] It's time to notify for 162
Sep 22 23:22:17.318 INFO Completion from [0] id:162 status:true
Sep 22 23:22:17.318 INFO [163/752] Repair commands completed
Sep 22 23:22:17.318 INFO Pop front: ReconcileIO { id: ReconciliationId(163), op: ExtentReopen { repair_id: ReconciliationId(163), extent_id: 184 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.318 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.318 INFO [0] received reconcile message
Sep 22 23:22:17.318 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(163), op: ExtentReopen { repair_id: ReconciliationId(163), extent_id: 184 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.318 INFO [0] client ExtentReopen { repair_id: ReconciliationId(163), extent_id: 184 }
Sep 22 23:22:17.318 INFO [1] received reconcile message
Sep 22 23:22:17.318 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(163), op: ExtentReopen { repair_id: ReconciliationId(163), extent_id: 184 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.318 INFO [1] client ExtentReopen { repair_id: ReconciliationId(163), extent_id: 184 }
Sep 22 23:22:17.318 INFO [2] received reconcile message
Sep 22 23:22:17.318 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(163), op: ExtentReopen { repair_id: ReconciliationId(163), extent_id: 184 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.318 INFO [2] client ExtentReopen { repair_id: ReconciliationId(163), extent_id: 184 }
Sep 22 23:22:17.319 DEBG 163 Reopen extent 184
Sep 22 23:22:17.319 DEBG 163 Reopen extent 184
Sep 22 23:22:17.320 DEBG 163 Reopen extent 184
Sep 22 23:22:17.320 DEBG [2] It's time to notify for 163
Sep 22 23:22:17.320 INFO Completion from [2] id:163 status:true
Sep 22 23:22:17.320 INFO [164/752] Repair commands completed
Sep 22 23:22:17.320 INFO Pop front: ReconcileIO { id: ReconciliationId(164), op: ExtentFlush { repair_id: ReconciliationId(164), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.320 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.320 INFO [0] received reconcile message
Sep 22 23:22:17.320 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(164), op: ExtentFlush { repair_id: ReconciliationId(164), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.320 INFO [0] client ExtentFlush { repair_id: ReconciliationId(164), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.320 INFO [1] received reconcile message
Sep 22 23:22:17.320 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(164), op: ExtentFlush { repair_id: ReconciliationId(164), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.320 INFO [1] client ExtentFlush { repair_id: ReconciliationId(164), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.321 INFO [2] received reconcile message
Sep 22 23:22:17.321 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(164), op: ExtentFlush { repair_id: ReconciliationId(164), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.321 INFO [2] client ExtentFlush { repair_id: ReconciliationId(164), extent_id: 41, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.321 DEBG 164 Flush extent 41 with f:2 g:2
Sep 22 23:22:17.321 DEBG Flush just extent 41 with f:2 and g:2
Sep 22 23:22:17.321 DEBG [1] It's time to notify for 164
Sep 22 23:22:17.321 INFO Completion from [1] id:164 status:true
Sep 22 23:22:17.321 INFO [165/752] Repair commands completed
Sep 22 23:22:17.321 INFO Pop front: ReconcileIO { id: ReconciliationId(165), op: ExtentClose { repair_id: ReconciliationId(165), extent_id: 41 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.321 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.321 INFO [0] received reconcile message
Sep 22 23:22:17.321 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(165), op: ExtentClose { repair_id: ReconciliationId(165), extent_id: 41 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.321 INFO [0] client ExtentClose { repair_id: ReconciliationId(165), extent_id: 41 }
Sep 22 23:22:17.321 INFO [1] received reconcile message
Sep 22 23:22:17.321 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(165), op: ExtentClose { repair_id: ReconciliationId(165), extent_id: 41 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.321 INFO [1] client ExtentClose { repair_id: ReconciliationId(165), extent_id: 41 }
Sep 22 23:22:17.321 INFO [2] received reconcile message
Sep 22 23:22:17.321 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(165), op: ExtentClose { repair_id: ReconciliationId(165), extent_id: 41 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.321 INFO [2] client ExtentClose { repair_id: ReconciliationId(165), extent_id: 41 }
Sep 22 23:22:17.321 DEBG 165 Close extent 41
Sep 22 23:22:17.321 DEBG 165 Close extent 41
Sep 22 23:22:17.322 DEBG 165 Close extent 41
Sep 22 23:22:17.322 DEBG [2] It's time to notify for 165
Sep 22 23:22:17.322 INFO Completion from [2] id:165 status:true
Sep 22 23:22:17.322 INFO [166/752] Repair commands completed
Sep 22 23:22:17.322 INFO Pop front: ReconcileIO { id: ReconciliationId(166), op: ExtentRepair { repair_id: ReconciliationId(166), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.322 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.322 INFO [0] received reconcile message
Sep 22 23:22:17.322 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(166), op: ExtentRepair { repair_id: ReconciliationId(166), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.322 INFO [0] client ExtentRepair { repair_id: ReconciliationId(166), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.322 INFO [0] Sending repair request ReconciliationId(166)
Sep 22 23:22:17.322 INFO [1] received reconcile message
Sep 22 23:22:17.322 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(166), op: ExtentRepair { repair_id: ReconciliationId(166), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.323 INFO [1] client ExtentRepair { repair_id: ReconciliationId(166), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.323 INFO [1] No action required ReconciliationId(166)
Sep 22 23:22:17.323 INFO [2] received reconcile message
Sep 22 23:22:17.323 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(166), op: ExtentRepair { repair_id: ReconciliationId(166), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.323 INFO [2] client ExtentRepair { repair_id: ReconciliationId(166), extent_id: 41, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.323 INFO [2] No action required ReconciliationId(166)
Sep 22 23:22:17.323 DEBG 166 Repair extent 41 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.323 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/029.copy"
Sep 22 23:22:17.350 DEBG [rc] retire 1076 clears [JobId(1075), JobId(1076)], : downstairs
Sep 22 23:22:17.350 INFO [lossy] skipping 1077
Sep 22 23:22:17.350 INFO [lossy] skipping 1077
Sep 22 23:22:17.356 DEBG Read      :1077 deps:[JobId(1076)] res:true
Sep 22 23:22:17.380 DEBG Flush     :1078 extent_limit None deps:[JobId(1077), JobId(1076)] res:true f:28 g:1
Sep 22 23:22:17.380 INFO [lossy] sleeping 1 second
Sep 22 23:22:17.386 INFO accepted connection, remote_addr: 127.0.0.1:43792, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.386 TRCE incoming request, uri: /extent/41/files, method: GET, req_id: 73c43dc5-1d03-4b46-8dfa-b037d87c4dda, remote_addr: 127.0.0.1:43792, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.386 INFO request completed, latency_us: 202, response_code: 200, uri: /extent/41/files, method: GET, req_id: 73c43dc5-1d03-4b46-8dfa-b037d87c4dda, remote_addr: 127.0.0.1:43792, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.386 INFO eid:41 Found repair files: ["029", "029.db"]
Sep 22 23:22:17.387 TRCE incoming request, uri: /newextent/41/data, method: GET, req_id: f53995b0-7e9e-4356-a4e9-a4f74c1c59b0, remote_addr: 127.0.0.1:43792, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.387 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/41/data, method: GET, req_id: f53995b0-7e9e-4356-a4e9-a4f74c1c59b0, remote_addr: 127.0.0.1:43792, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.392 TRCE incoming request, uri: /newextent/41/db, method: GET, req_id: e9d154ec-08fb-491f-bae0-1cbda091d4e1, remote_addr: 127.0.0.1:43792, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.392 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/41/db, method: GET, req_id: e9d154ec-08fb-491f-bae0-1cbda091d4e1, remote_addr: 127.0.0.1:43792, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.393 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/029.copy" to "/tmp/downstairs-zrMnlo6G/00/000/029.replace"
Sep 22 23:22:17.393 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.394 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/029.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.394 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/029"
Sep 22 23:22:17.394 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/029.db"
Sep 22 23:22:17.394 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.394 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/029.replace" to "/tmp/downstairs-zrMnlo6G/00/000/029.completed"
Sep 22 23:22:17.394 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.394 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.394 DEBG [0] It's time to notify for 166
Sep 22 23:22:17.395 INFO Completion from [0] id:166 status:true
Sep 22 23:22:17.395 INFO [167/752] Repair commands completed
Sep 22 23:22:17.395 INFO Pop front: ReconcileIO { id: ReconciliationId(167), op: ExtentReopen { repair_id: ReconciliationId(167), extent_id: 41 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.395 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.395 INFO [0] received reconcile message
Sep 22 23:22:17.395 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(167), op: ExtentReopen { repair_id: ReconciliationId(167), extent_id: 41 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.395 INFO [0] client ExtentReopen { repair_id: ReconciliationId(167), extent_id: 41 }
Sep 22 23:22:17.395 INFO [1] received reconcile message
Sep 22 23:22:17.395 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(167), op: ExtentReopen { repair_id: ReconciliationId(167), extent_id: 41 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.395 INFO [1] client ExtentReopen { repair_id: ReconciliationId(167), extent_id: 41 }
Sep 22 23:22:17.395 INFO [2] received reconcile message
Sep 22 23:22:17.395 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(167), op: ExtentReopen { repair_id: ReconciliationId(167), extent_id: 41 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.395 INFO [2] client ExtentReopen { repair_id: ReconciliationId(167), extent_id: 41 }
Sep 22 23:22:17.395 DEBG 167 Reopen extent 41
Sep 22 23:22:17.396 DEBG 167 Reopen extent 41
Sep 22 23:22:17.396 DEBG 167 Reopen extent 41
Sep 22 23:22:17.397 DEBG [2] It's time to notify for 167
Sep 22 23:22:17.397 INFO Completion from [2] id:167 status:true
Sep 22 23:22:17.397 INFO [168/752] Repair commands completed
Sep 22 23:22:17.397 INFO Pop front: ReconcileIO { id: ReconciliationId(168), op: ExtentFlush { repair_id: ReconciliationId(168), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.397 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.397 INFO [0] received reconcile message
Sep 22 23:22:17.397 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(168), op: ExtentFlush { repair_id: ReconciliationId(168), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.397 INFO [0] client ExtentFlush { repair_id: ReconciliationId(168), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.397 INFO [1] received reconcile message
Sep 22 23:22:17.397 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(168), op: ExtentFlush { repair_id: ReconciliationId(168), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.397 INFO [1] client ExtentFlush { repair_id: ReconciliationId(168), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.397 INFO [2] received reconcile message
Sep 22 23:22:17.397 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(168), op: ExtentFlush { repair_id: ReconciliationId(168), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.397 INFO [2] client ExtentFlush { repair_id: ReconciliationId(168), extent_id: 77, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.397 DEBG 168 Flush extent 77 with f:2 g:2
Sep 22 23:22:17.397 DEBG Flush just extent 77 with f:2 and g:2
Sep 22 23:22:17.397 DEBG [1] It's time to notify for 168
Sep 22 23:22:17.397 INFO Completion from [1] id:168 status:true
Sep 22 23:22:17.397 INFO [169/752] Repair commands completed
Sep 22 23:22:17.397 INFO Pop front: ReconcileIO { id: ReconciliationId(169), op: ExtentClose { repair_id: ReconciliationId(169), extent_id: 77 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.397 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.397 INFO [0] received reconcile message
Sep 22 23:22:17.397 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(169), op: ExtentClose { repair_id: ReconciliationId(169), extent_id: 77 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.398 INFO [0] client ExtentClose { repair_id: ReconciliationId(169), extent_id: 77 }
Sep 22 23:22:17.398 INFO [1] received reconcile message
Sep 22 23:22:17.398 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(169), op: ExtentClose { repair_id: ReconciliationId(169), extent_id: 77 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.398 INFO [1] client ExtentClose { repair_id: ReconciliationId(169), extent_id: 77 }
Sep 22 23:22:17.398 INFO [2] received reconcile message
Sep 22 23:22:17.398 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(169), op: ExtentClose { repair_id: ReconciliationId(169), extent_id: 77 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.398 INFO [2] client ExtentClose { repair_id: ReconciliationId(169), extent_id: 77 }
Sep 22 23:22:17.398 DEBG 169 Close extent 77
Sep 22 23:22:17.398 DEBG 169 Close extent 77
Sep 22 23:22:17.398 DEBG 169 Close extent 77
Sep 22 23:22:17.399 DEBG [2] It's time to notify for 169
Sep 22 23:22:17.399 INFO Completion from [2] id:169 status:true
Sep 22 23:22:17.399 INFO [170/752] Repair commands completed
Sep 22 23:22:17.399 INFO Pop front: ReconcileIO { id: ReconciliationId(170), op: ExtentRepair { repair_id: ReconciliationId(170), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.399 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.399 INFO [0] received reconcile message
Sep 22 23:22:17.399 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(170), op: ExtentRepair { repair_id: ReconciliationId(170), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.399 INFO [0] client ExtentRepair { repair_id: ReconciliationId(170), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.399 INFO [0] Sending repair request ReconciliationId(170)
Sep 22 23:22:17.399 INFO [1] received reconcile message
Sep 22 23:22:17.399 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(170), op: ExtentRepair { repair_id: ReconciliationId(170), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.399 INFO [1] client ExtentRepair { repair_id: ReconciliationId(170), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.399 INFO [1] No action required ReconciliationId(170)
Sep 22 23:22:17.399 INFO [2] received reconcile message
Sep 22 23:22:17.399 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(170), op: ExtentRepair { repair_id: ReconciliationId(170), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.399 INFO [2] client ExtentRepair { repair_id: ReconciliationId(170), extent_id: 77, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.399 INFO [2] No action required ReconciliationId(170)
Sep 22 23:22:17.399 DEBG 170 Repair extent 77 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.399 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/04D.copy"
Sep 22 23:22:17.463 INFO accepted connection, remote_addr: 127.0.0.1:53871, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.464 TRCE incoming request, uri: /extent/77/files, method: GET, req_id: fccb43e6-ddcb-4895-b0a8-ead3b604cd41, remote_addr: 127.0.0.1:53871, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.464 INFO request completed, latency_us: 203, response_code: 200, uri: /extent/77/files, method: GET, req_id: fccb43e6-ddcb-4895-b0a8-ead3b604cd41, remote_addr: 127.0.0.1:53871, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.464 INFO eid:77 Found repair files: ["04D", "04D.db"]
Sep 22 23:22:17.464 TRCE incoming request, uri: /newextent/77/data, method: GET, req_id: 8e26ad80-865f-44e4-b4e3-2d6a13dc62f8, remote_addr: 127.0.0.1:53871, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.465 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/77/data, method: GET, req_id: 8e26ad80-865f-44e4-b4e3-2d6a13dc62f8, remote_addr: 127.0.0.1:53871, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.469 TRCE incoming request, uri: /newextent/77/db, method: GET, req_id: 7a2b16e6-2bbf-43a0-b9c2-0f3a086191b5, remote_addr: 127.0.0.1:53871, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.470 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/77/db, method: GET, req_id: 7a2b16e6-2bbf-43a0-b9c2-0f3a086191b5, remote_addr: 127.0.0.1:53871, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.471 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/04D.copy" to "/tmp/downstairs-zrMnlo6G/00/000/04D.replace"
Sep 22 23:22:17.471 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.472 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/04D.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.472 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04D"
Sep 22 23:22:17.472 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04D.db"
Sep 22 23:22:17.472 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.472 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/04D.replace" to "/tmp/downstairs-zrMnlo6G/00/000/04D.completed"
Sep 22 23:22:17.472 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.472 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.472 DEBG [0] It's time to notify for 170
Sep 22 23:22:17.473 INFO Completion from [0] id:170 status:true
Sep 22 23:22:17.473 INFO [171/752] Repair commands completed
Sep 22 23:22:17.473 INFO Pop front: ReconcileIO { id: ReconciliationId(171), op: ExtentReopen { repair_id: ReconciliationId(171), extent_id: 77 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.473 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.473 INFO [0] received reconcile message
Sep 22 23:22:17.473 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(171), op: ExtentReopen { repair_id: ReconciliationId(171), extent_id: 77 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.473 INFO [0] client ExtentReopen { repair_id: ReconciliationId(171), extent_id: 77 }
Sep 22 23:22:17.473 INFO [1] received reconcile message
Sep 22 23:22:17.473 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(171), op: ExtentReopen { repair_id: ReconciliationId(171), extent_id: 77 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.473 INFO [1] client ExtentReopen { repair_id: ReconciliationId(171), extent_id: 77 }
Sep 22 23:22:17.473 INFO [2] received reconcile message
Sep 22 23:22:17.473 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(171), op: ExtentReopen { repair_id: ReconciliationId(171), extent_id: 77 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.473 INFO [2] client ExtentReopen { repair_id: ReconciliationId(171), extent_id: 77 }
Sep 22 23:22:17.473 DEBG 171 Reopen extent 77
Sep 22 23:22:17.474 DEBG 171 Reopen extent 77
Sep 22 23:22:17.474 DEBG 171 Reopen extent 77
Sep 22 23:22:17.475 DEBG [2] It's time to notify for 171
Sep 22 23:22:17.475 INFO Completion from [2] id:171 status:true
Sep 22 23:22:17.475 INFO [172/752] Repair commands completed
Sep 22 23:22:17.475 INFO Pop front: ReconcileIO { id: ReconciliationId(172), op: ExtentFlush { repair_id: ReconciliationId(172), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.475 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.475 INFO [0] received reconcile message
Sep 22 23:22:17.475 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(172), op: ExtentFlush { repair_id: ReconciliationId(172), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.475 INFO [0] client ExtentFlush { repair_id: ReconciliationId(172), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.475 INFO [1] received reconcile message
Sep 22 23:22:17.475 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(172), op: ExtentFlush { repair_id: ReconciliationId(172), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.475 INFO [1] client ExtentFlush { repair_id: ReconciliationId(172), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.475 INFO [2] received reconcile message
Sep 22 23:22:17.475 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(172), op: ExtentFlush { repair_id: ReconciliationId(172), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.475 INFO [2] client ExtentFlush { repair_id: ReconciliationId(172), extent_id: 107, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.475 DEBG 172 Flush extent 107 with f:2 g:2
Sep 22 23:22:17.475 DEBG Flush just extent 107 with f:2 and g:2
Sep 22 23:22:17.475 DEBG [1] It's time to notify for 172
Sep 22 23:22:17.475 INFO Completion from [1] id:172 status:true
Sep 22 23:22:17.475 INFO [173/752] Repair commands completed
Sep 22 23:22:17.475 INFO Pop front: ReconcileIO { id: ReconciliationId(173), op: ExtentClose { repair_id: ReconciliationId(173), extent_id: 107 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.475 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.475 INFO [0] received reconcile message
Sep 22 23:22:17.476 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(173), op: ExtentClose { repair_id: ReconciliationId(173), extent_id: 107 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.476 INFO [0] client ExtentClose { repair_id: ReconciliationId(173), extent_id: 107 }
Sep 22 23:22:17.476 INFO [1] received reconcile message
Sep 22 23:22:17.476 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(173), op: ExtentClose { repair_id: ReconciliationId(173), extent_id: 107 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.476 INFO [1] client ExtentClose { repair_id: ReconciliationId(173), extent_id: 107 }
Sep 22 23:22:17.476 INFO [2] received reconcile message
Sep 22 23:22:17.476 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(173), op: ExtentClose { repair_id: ReconciliationId(173), extent_id: 107 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.476 INFO [2] client ExtentClose { repair_id: ReconciliationId(173), extent_id: 107 }
Sep 22 23:22:17.476 DEBG 173 Close extent 107
Sep 22 23:22:17.476 DEBG 173 Close extent 107
Sep 22 23:22:17.476 DEBG 173 Close extent 107
Sep 22 23:22:17.477 DEBG [2] It's time to notify for 173
Sep 22 23:22:17.477 INFO Completion from [2] id:173 status:true
Sep 22 23:22:17.477 INFO [174/752] Repair commands completed
Sep 22 23:22:17.477 INFO Pop front: ReconcileIO { id: ReconciliationId(174), op: ExtentRepair { repair_id: ReconciliationId(174), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.477 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.477 INFO [0] received reconcile message
Sep 22 23:22:17.477 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(174), op: ExtentRepair { repair_id: ReconciliationId(174), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.477 INFO [0] client ExtentRepair { repair_id: ReconciliationId(174), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.477 INFO [0] Sending repair request ReconciliationId(174)
Sep 22 23:22:17.477 INFO [1] received reconcile message
Sep 22 23:22:17.477 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(174), op: ExtentRepair { repair_id: ReconciliationId(174), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.477 INFO [1] client ExtentRepair { repair_id: ReconciliationId(174), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.477 INFO [1] No action required ReconciliationId(174)
Sep 22 23:22:17.477 INFO [2] received reconcile message
Sep 22 23:22:17.477 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(174), op: ExtentRepair { repair_id: ReconciliationId(174), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.477 INFO [2] client ExtentRepair { repair_id: ReconciliationId(174), extent_id: 107, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.477 INFO [2] No action required ReconciliationId(174)
Sep 22 23:22:17.477 DEBG 174 Repair extent 107 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.477 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/06B.copy"
Sep 22 23:22:17.544 INFO accepted connection, remote_addr: 127.0.0.1:34959, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.544 TRCE incoming request, uri: /extent/107/files, method: GET, req_id: 437d39af-0e2e-4906-869b-239637735a2e, remote_addr: 127.0.0.1:34959, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.544 INFO request completed, latency_us: 280, response_code: 200, uri: /extent/107/files, method: GET, req_id: 437d39af-0e2e-4906-869b-239637735a2e, remote_addr: 127.0.0.1:34959, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.545 INFO eid:107 Found repair files: ["06B", "06B.db"]
Sep 22 23:22:17.545 TRCE incoming request, uri: /newextent/107/data, method: GET, req_id: 02cda503-d1e3-42a4-b6b5-4c97685a8658, remote_addr: 127.0.0.1:34959, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.545 INFO request completed, latency_us: 367, response_code: 200, uri: /newextent/107/data, method: GET, req_id: 02cda503-d1e3-42a4-b6b5-4c97685a8658, remote_addr: 127.0.0.1:34959, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.550 TRCE incoming request, uri: /newextent/107/db, method: GET, req_id: d687edd0-129d-497d-ab9e-21dd3002cab1, remote_addr: 127.0.0.1:34959, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.551 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/107/db, method: GET, req_id: d687edd0-129d-497d-ab9e-21dd3002cab1, remote_addr: 127.0.0.1:34959, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.552 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/06B.copy" to "/tmp/downstairs-zrMnlo6G/00/000/06B.replace"
Sep 22 23:22:17.552 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.553 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/06B.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.554 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06B"
Sep 22 23:22:17.554 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06B.db"
Sep 22 23:22:17.554 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.554 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/06B.replace" to "/tmp/downstairs-zrMnlo6G/00/000/06B.completed"
Sep 22 23:22:17.554 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.554 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.554 DEBG [0] It's time to notify for 174
Sep 22 23:22:17.554 INFO Completion from [0] id:174 status:true
Sep 22 23:22:17.554 INFO [175/752] Repair commands completed
Sep 22 23:22:17.554 INFO Pop front: ReconcileIO { id: ReconciliationId(175), op: ExtentReopen { repair_id: ReconciliationId(175), extent_id: 107 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.554 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.554 INFO [0] received reconcile message
Sep 22 23:22:17.554 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(175), op: ExtentReopen { repair_id: ReconciliationId(175), extent_id: 107 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.554 INFO [0] client ExtentReopen { repair_id: ReconciliationId(175), extent_id: 107 }
Sep 22 23:22:17.554 INFO [1] received reconcile message
Sep 22 23:22:17.555 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(175), op: ExtentReopen { repair_id: ReconciliationId(175), extent_id: 107 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.555 INFO [1] client ExtentReopen { repair_id: ReconciliationId(175), extent_id: 107 }
Sep 22 23:22:17.555 INFO [2] received reconcile message
Sep 22 23:22:17.555 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(175), op: ExtentReopen { repair_id: ReconciliationId(175), extent_id: 107 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.555 INFO [2] client ExtentReopen { repair_id: ReconciliationId(175), extent_id: 107 }
Sep 22 23:22:17.555 DEBG 175 Reopen extent 107
Sep 22 23:22:17.556 DEBG 175 Reopen extent 107
Sep 22 23:22:17.556 DEBG 175 Reopen extent 107
Sep 22 23:22:17.557 DEBG [2] It's time to notify for 175
Sep 22 23:22:17.557 INFO Completion from [2] id:175 status:true
Sep 22 23:22:17.557 INFO [176/752] Repair commands completed
Sep 22 23:22:17.557 INFO Pop front: ReconcileIO { id: ReconciliationId(176), op: ExtentFlush { repair_id: ReconciliationId(176), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.557 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.557 INFO [0] received reconcile message
Sep 22 23:22:17.557 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(176), op: ExtentFlush { repair_id: ReconciliationId(176), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.557 INFO [0] client ExtentFlush { repair_id: ReconciliationId(176), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.557 INFO [1] received reconcile message
Sep 22 23:22:17.557 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(176), op: ExtentFlush { repair_id: ReconciliationId(176), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.557 INFO [1] client ExtentFlush { repair_id: ReconciliationId(176), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.557 INFO [2] received reconcile message
Sep 22 23:22:17.557 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(176), op: ExtentFlush { repair_id: ReconciliationId(176), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.557 INFO [2] client ExtentFlush { repair_id: ReconciliationId(176), extent_id: 64, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.557 DEBG 176 Flush extent 64 with f:2 g:2
Sep 22 23:22:17.557 DEBG Flush just extent 64 with f:2 and g:2
Sep 22 23:22:17.557 DEBG [1] It's time to notify for 176
Sep 22 23:22:17.557 INFO Completion from [1] id:176 status:true
Sep 22 23:22:17.557 INFO [177/752] Repair commands completed
Sep 22 23:22:17.557 INFO Pop front: ReconcileIO { id: ReconciliationId(177), op: ExtentClose { repair_id: ReconciliationId(177), extent_id: 64 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.557 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.557 INFO [0] received reconcile message
Sep 22 23:22:17.557 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(177), op: ExtentClose { repair_id: ReconciliationId(177), extent_id: 64 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.557 INFO [0] client ExtentClose { repair_id: ReconciliationId(177), extent_id: 64 }
Sep 22 23:22:17.558 INFO [1] received reconcile message
Sep 22 23:22:17.558 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(177), op: ExtentClose { repair_id: ReconciliationId(177), extent_id: 64 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.558 INFO [1] client ExtentClose { repair_id: ReconciliationId(177), extent_id: 64 }
Sep 22 23:22:17.558 INFO [2] received reconcile message
Sep 22 23:22:17.558 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(177), op: ExtentClose { repair_id: ReconciliationId(177), extent_id: 64 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.558 INFO [2] client ExtentClose { repair_id: ReconciliationId(177), extent_id: 64 }
Sep 22 23:22:17.558 DEBG 177 Close extent 64
Sep 22 23:22:17.558 DEBG 177 Close extent 64
Sep 22 23:22:17.558 DEBG 177 Close extent 64
Sep 22 23:22:17.559 DEBG [2] It's time to notify for 177
Sep 22 23:22:17.559 INFO Completion from [2] id:177 status:true
Sep 22 23:22:17.559 INFO [178/752] Repair commands completed
Sep 22 23:22:17.559 INFO Pop front: ReconcileIO { id: ReconciliationId(178), op: ExtentRepair { repair_id: ReconciliationId(178), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.559 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.559 INFO [0] received reconcile message
Sep 22 23:22:17.559 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(178), op: ExtentRepair { repair_id: ReconciliationId(178), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.559 INFO [0] client ExtentRepair { repair_id: ReconciliationId(178), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.559 INFO [0] Sending repair request ReconciliationId(178)
Sep 22 23:22:17.559 INFO [1] received reconcile message
Sep 22 23:22:17.559 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(178), op: ExtentRepair { repair_id: ReconciliationId(178), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.559 INFO [1] client ExtentRepair { repair_id: ReconciliationId(178), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.559 INFO [1] No action required ReconciliationId(178)
Sep 22 23:22:17.559 INFO [2] received reconcile message
Sep 22 23:22:17.559 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(178), op: ExtentRepair { repair_id: ReconciliationId(178), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.559 INFO [2] client ExtentRepair { repair_id: ReconciliationId(178), extent_id: 64, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.559 INFO [2] No action required ReconciliationId(178)
Sep 22 23:22:17.559 DEBG 178 Repair extent 64 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.559 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/040.copy"
Sep 22 23:22:17.622 INFO accepted connection, remote_addr: 127.0.0.1:64907, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.622 TRCE incoming request, uri: /extent/64/files, method: GET, req_id: 39b232b0-a0e4-4484-a42e-264c8fd1ab91, remote_addr: 127.0.0.1:64907, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.622 INFO request completed, latency_us: 252, response_code: 200, uri: /extent/64/files, method: GET, req_id: 39b232b0-a0e4-4484-a42e-264c8fd1ab91, remote_addr: 127.0.0.1:64907, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.623 INFO eid:64 Found repair files: ["040", "040.db"]
Sep 22 23:22:17.623 TRCE incoming request, uri: /newextent/64/data, method: GET, req_id: 9f3b4fe9-4a06-42c8-9149-947d846f8b64, remote_addr: 127.0.0.1:64907, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.623 INFO request completed, latency_us: 341, response_code: 200, uri: /newextent/64/data, method: GET, req_id: 9f3b4fe9-4a06-42c8-9149-947d846f8b64, remote_addr: 127.0.0.1:64907, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.628 TRCE incoming request, uri: /newextent/64/db, method: GET, req_id: 67d4f62a-358c-4da8-9fbc-258047443558, remote_addr: 127.0.0.1:64907, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.629 INFO request completed, latency_us: 288, response_code: 200, uri: /newextent/64/db, method: GET, req_id: 67d4f62a-358c-4da8-9fbc-258047443558, remote_addr: 127.0.0.1:64907, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.630 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/040.copy" to "/tmp/downstairs-zrMnlo6G/00/000/040.replace"
Sep 22 23:22:17.630 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.630 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/040.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.631 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/040"
Sep 22 23:22:17.631 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/040.db"
Sep 22 23:22:17.631 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.631 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/040.replace" to "/tmp/downstairs-zrMnlo6G/00/000/040.completed"
Sep 22 23:22:17.631 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.631 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.631 DEBG [0] It's time to notify for 178
Sep 22 23:22:17.631 INFO Completion from [0] id:178 status:true
Sep 22 23:22:17.631 INFO [179/752] Repair commands completed
Sep 22 23:22:17.631 INFO Pop front: ReconcileIO { id: ReconciliationId(179), op: ExtentReopen { repair_id: ReconciliationId(179), extent_id: 64 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.631 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.631 INFO [0] received reconcile message
Sep 22 23:22:17.631 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(179), op: ExtentReopen { repair_id: ReconciliationId(179), extent_id: 64 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.631 INFO [0] client ExtentReopen { repair_id: ReconciliationId(179), extent_id: 64 }
Sep 22 23:22:17.631 INFO [1] received reconcile message
Sep 22 23:22:17.631 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(179), op: ExtentReopen { repair_id: ReconciliationId(179), extent_id: 64 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.631 INFO [1] client ExtentReopen { repair_id: ReconciliationId(179), extent_id: 64 }
Sep 22 23:22:17.632 INFO [2] received reconcile message
Sep 22 23:22:17.632 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(179), op: ExtentReopen { repair_id: ReconciliationId(179), extent_id: 64 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.632 INFO [2] client ExtentReopen { repair_id: ReconciliationId(179), extent_id: 64 }
Sep 22 23:22:17.632 DEBG 179 Reopen extent 64
Sep 22 23:22:17.632 DEBG 179 Reopen extent 64
Sep 22 23:22:17.633 DEBG 179 Reopen extent 64
Sep 22 23:22:17.633 DEBG [2] It's time to notify for 179
Sep 22 23:22:17.634 INFO Completion from [2] id:179 status:true
Sep 22 23:22:17.634 INFO [180/752] Repair commands completed
Sep 22 23:22:17.634 INFO Pop front: ReconcileIO { id: ReconciliationId(180), op: ExtentFlush { repair_id: ReconciliationId(180), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.634 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.634 INFO [0] received reconcile message
Sep 22 23:22:17.634 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(180), op: ExtentFlush { repair_id: ReconciliationId(180), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.634 INFO [0] client ExtentFlush { repair_id: ReconciliationId(180), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.634 INFO [1] received reconcile message
Sep 22 23:22:17.634 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(180), op: ExtentFlush { repair_id: ReconciliationId(180), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.634 INFO [1] client ExtentFlush { repair_id: ReconciliationId(180), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.634 INFO [2] received reconcile message
Sep 22 23:22:17.634 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(180), op: ExtentFlush { repair_id: ReconciliationId(180), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.634 INFO [2] client ExtentFlush { repair_id: ReconciliationId(180), extent_id: 171, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.634 DEBG 180 Flush extent 171 with f:2 g:2
Sep 22 23:22:17.634 DEBG Flush just extent 171 with f:2 and g:2
Sep 22 23:22:17.634 DEBG [1] It's time to notify for 180
Sep 22 23:22:17.634 INFO Completion from [1] id:180 status:true
Sep 22 23:22:17.634 INFO [181/752] Repair commands completed
Sep 22 23:22:17.634 INFO Pop front: ReconcileIO { id: ReconciliationId(181), op: ExtentClose { repair_id: ReconciliationId(181), extent_id: 171 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.634 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.634 INFO [0] received reconcile message
Sep 22 23:22:17.634 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(181), op: ExtentClose { repair_id: ReconciliationId(181), extent_id: 171 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.634 INFO [0] client ExtentClose { repair_id: ReconciliationId(181), extent_id: 171 }
Sep 22 23:22:17.634 INFO [1] received reconcile message
Sep 22 23:22:17.634 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(181), op: ExtentClose { repair_id: ReconciliationId(181), extent_id: 171 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.634 INFO [1] client ExtentClose { repair_id: ReconciliationId(181), extent_id: 171 }
Sep 22 23:22:17.634 INFO [2] received reconcile message
Sep 22 23:22:17.634 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(181), op: ExtentClose { repair_id: ReconciliationId(181), extent_id: 171 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.634 INFO [2] client ExtentClose { repair_id: ReconciliationId(181), extent_id: 171 }
Sep 22 23:22:17.635 DEBG 181 Close extent 171
Sep 22 23:22:17.635 DEBG 181 Close extent 171
Sep 22 23:22:17.635 DEBG 181 Close extent 171
Sep 22 23:22:17.635 DEBG [2] It's time to notify for 181
Sep 22 23:22:17.636 INFO Completion from [2] id:181 status:true
Sep 22 23:22:17.636 INFO [182/752] Repair commands completed
Sep 22 23:22:17.636 INFO Pop front: ReconcileIO { id: ReconciliationId(182), op: ExtentRepair { repair_id: ReconciliationId(182), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.636 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.636 INFO [0] received reconcile message
Sep 22 23:22:17.636 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(182), op: ExtentRepair { repair_id: ReconciliationId(182), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.636 INFO [0] client ExtentRepair { repair_id: ReconciliationId(182), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.636 INFO [0] Sending repair request ReconciliationId(182)
Sep 22 23:22:17.636 INFO [1] received reconcile message
Sep 22 23:22:17.636 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(182), op: ExtentRepair { repair_id: ReconciliationId(182), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.636 INFO [1] client ExtentRepair { repair_id: ReconciliationId(182), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.636 INFO [1] No action required ReconciliationId(182)
Sep 22 23:22:17.636 INFO [2] received reconcile message
Sep 22 23:22:17.636 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(182), op: ExtentRepair { repair_id: ReconciliationId(182), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.636 INFO [2] client ExtentRepair { repair_id: ReconciliationId(182), extent_id: 171, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.636 INFO [2] No action required ReconciliationId(182)
Sep 22 23:22:17.636 DEBG 182 Repair extent 171 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.636 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0AB.copy"
Sep 22 23:22:17.697 INFO accepted connection, remote_addr: 127.0.0.1:33985, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.698 TRCE incoming request, uri: /extent/171/files, method: GET, req_id: ece9d917-df70-4a82-b1a0-00cb79bd2cde, remote_addr: 127.0.0.1:33985, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.698 INFO request completed, latency_us: 205, response_code: 200, uri: /extent/171/files, method: GET, req_id: ece9d917-df70-4a82-b1a0-00cb79bd2cde, remote_addr: 127.0.0.1:33985, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.698 INFO eid:171 Found repair files: ["0AB", "0AB.db"]
Sep 22 23:22:17.698 TRCE incoming request, uri: /newextent/171/data, method: GET, req_id: d7789424-3ba1-4ef2-93e2-d1c874159df6, remote_addr: 127.0.0.1:33985, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.699 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/171/data, method: GET, req_id: d7789424-3ba1-4ef2-93e2-d1c874159df6, remote_addr: 127.0.0.1:33985, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.704 TRCE incoming request, uri: /newextent/171/db, method: GET, req_id: 8807af28-ee9c-42c4-9ffe-3e41f34a06d2, remote_addr: 127.0.0.1:33985, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.704 INFO request completed, latency_us: 289, response_code: 200, uri: /newextent/171/db, method: GET, req_id: 8807af28-ee9c-42c4-9ffe-3e41f34a06d2, remote_addr: 127.0.0.1:33985, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.705 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0AB.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0AB.replace"
Sep 22 23:22:17.705 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.706 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0AB.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.706 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AB"
Sep 22 23:22:17.706 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AB.db"
Sep 22 23:22:17.706 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.706 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0AB.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0AB.completed"
Sep 22 23:22:17.706 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.706 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.706 DEBG [0] It's time to notify for 182
Sep 22 23:22:17.707 INFO Completion from [0] id:182 status:true
Sep 22 23:22:17.707 INFO [183/752] Repair commands completed
Sep 22 23:22:17.707 INFO Pop front: ReconcileIO { id: ReconciliationId(183), op: ExtentReopen { repair_id: ReconciliationId(183), extent_id: 171 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.707 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.707 INFO [0] received reconcile message
Sep 22 23:22:17.707 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(183), op: ExtentReopen { repair_id: ReconciliationId(183), extent_id: 171 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.707 INFO [0] client ExtentReopen { repair_id: ReconciliationId(183), extent_id: 171 }
Sep 22 23:22:17.707 INFO [1] received reconcile message
Sep 22 23:22:17.707 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(183), op: ExtentReopen { repair_id: ReconciliationId(183), extent_id: 171 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.707 INFO [1] client ExtentReopen { repair_id: ReconciliationId(183), extent_id: 171 }
Sep 22 23:22:17.707 INFO [2] received reconcile message
Sep 22 23:22:17.707 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(183), op: ExtentReopen { repair_id: ReconciliationId(183), extent_id: 171 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.707 INFO [2] client ExtentReopen { repair_id: ReconciliationId(183), extent_id: 171 }
Sep 22 23:22:17.707 DEBG 183 Reopen extent 171
Sep 22 23:22:17.708 DEBG 183 Reopen extent 171
Sep 22 23:22:17.708 DEBG 183 Reopen extent 171
Sep 22 23:22:17.709 DEBG [2] It's time to notify for 183
Sep 22 23:22:17.709 INFO Completion from [2] id:183 status:true
Sep 22 23:22:17.709 INFO [184/752] Repair commands completed
Sep 22 23:22:17.709 INFO Pop front: ReconcileIO { id: ReconciliationId(184), op: ExtentFlush { repair_id: ReconciliationId(184), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.709 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.709 INFO [0] received reconcile message
Sep 22 23:22:17.709 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(184), op: ExtentFlush { repair_id: ReconciliationId(184), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.709 INFO [0] client ExtentFlush { repair_id: ReconciliationId(184), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.709 INFO [1] received reconcile message
Sep 22 23:22:17.709 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(184), op: ExtentFlush { repair_id: ReconciliationId(184), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.709 INFO [1] client ExtentFlush { repair_id: ReconciliationId(184), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.709 INFO [2] received reconcile message
Sep 22 23:22:17.709 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(184), op: ExtentFlush { repair_id: ReconciliationId(184), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.709 INFO [2] client ExtentFlush { repair_id: ReconciliationId(184), extent_id: 165, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.709 DEBG 184 Flush extent 165 with f:2 g:2
Sep 22 23:22:17.709 DEBG Flush just extent 165 with f:2 and g:2
Sep 22 23:22:17.709 DEBG [1] It's time to notify for 184
Sep 22 23:22:17.709 INFO Completion from [1] id:184 status:true
Sep 22 23:22:17.709 INFO [185/752] Repair commands completed
Sep 22 23:22:17.709 INFO Pop front: ReconcileIO { id: ReconciliationId(185), op: ExtentClose { repair_id: ReconciliationId(185), extent_id: 165 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.709 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.709 INFO [0] received reconcile message
Sep 22 23:22:17.709 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(185), op: ExtentClose { repair_id: ReconciliationId(185), extent_id: 165 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.709 INFO [0] client ExtentClose { repair_id: ReconciliationId(185), extent_id: 165 }
Sep 22 23:22:17.709 INFO [1] received reconcile message
Sep 22 23:22:17.709 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(185), op: ExtentClose { repair_id: ReconciliationId(185), extent_id: 165 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.709 INFO [1] client ExtentClose { repair_id: ReconciliationId(185), extent_id: 165 }
Sep 22 23:22:17.709 INFO [2] received reconcile message
Sep 22 23:22:17.710 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(185), op: ExtentClose { repair_id: ReconciliationId(185), extent_id: 165 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.710 INFO [2] client ExtentClose { repair_id: ReconciliationId(185), extent_id: 165 }
Sep 22 23:22:17.710 DEBG 185 Close extent 165
Sep 22 23:22:17.710 DEBG 185 Close extent 165
Sep 22 23:22:17.710 DEBG 185 Close extent 165
Sep 22 23:22:17.711 DEBG [2] It's time to notify for 185
Sep 22 23:22:17.711 INFO Completion from [2] id:185 status:true
Sep 22 23:22:17.711 INFO [186/752] Repair commands completed
Sep 22 23:22:17.711 INFO Pop front: ReconcileIO { id: ReconciliationId(186), op: ExtentRepair { repair_id: ReconciliationId(186), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.711 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.711 INFO [0] received reconcile message
Sep 22 23:22:17.711 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(186), op: ExtentRepair { repair_id: ReconciliationId(186), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.711 INFO [0] client ExtentRepair { repair_id: ReconciliationId(186), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.711 INFO [0] Sending repair request ReconciliationId(186)
Sep 22 23:22:17.711 INFO [1] received reconcile message
Sep 22 23:22:17.711 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(186), op: ExtentRepair { repair_id: ReconciliationId(186), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.711 INFO [1] client ExtentRepair { repair_id: ReconciliationId(186), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.711 INFO [1] No action required ReconciliationId(186)
Sep 22 23:22:17.711 INFO [2] received reconcile message
Sep 22 23:22:17.711 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(186), op: ExtentRepair { repair_id: ReconciliationId(186), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.711 INFO [2] client ExtentRepair { repair_id: ReconciliationId(186), extent_id: 165, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.711 INFO [2] No action required ReconciliationId(186)
Sep 22 23:22:17.711 DEBG 186 Repair extent 165 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.711 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0A5.copy"
Sep 22 23:22:17.759 DEBG [2] Read AckReady 1077, : downstairs
Sep 22 23:22:17.760 DEBG up_ds_listen was notified
Sep 22 23:22:17.760 DEBG up_ds_listen process 1077
Sep 22 23:22:17.760 DEBG [A] ack job 1077:78, : downstairs
Sep 22 23:22:17.775 INFO accepted connection, remote_addr: 127.0.0.1:61309, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.776 TRCE incoming request, uri: /extent/165/files, method: GET, req_id: 52ea5135-a5df-4d57-9ef4-d39aa05c4dfe, remote_addr: 127.0.0.1:61309, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.776 INFO request completed, latency_us: 193, response_code: 200, uri: /extent/165/files, method: GET, req_id: 52ea5135-a5df-4d57-9ef4-d39aa05c4dfe, remote_addr: 127.0.0.1:61309, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.776 INFO eid:165 Found repair files: ["0A5", "0A5.db"]
Sep 22 23:22:17.776 TRCE incoming request, uri: /newextent/165/data, method: GET, req_id: 2c1d08f6-50fb-4c17-b32e-7c94d4562d1e, remote_addr: 127.0.0.1:61309, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.777 INFO request completed, latency_us: 309, response_code: 200, uri: /newextent/165/data, method: GET, req_id: 2c1d08f6-50fb-4c17-b32e-7c94d4562d1e, remote_addr: 127.0.0.1:61309, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.782 TRCE incoming request, uri: /newextent/165/db, method: GET, req_id: 42d18f09-19f9-49ac-b6b9-96897066a57b, remote_addr: 127.0.0.1:61309, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.782 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/165/db, method: GET, req_id: 42d18f09-19f9-49ac-b6b9-96897066a57b, remote_addr: 127.0.0.1:61309, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.783 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0A5.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0A5.replace"
Sep 22 23:22:17.783 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.784 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0A5.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.784 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A5"
Sep 22 23:22:17.784 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A5.db"
Sep 22 23:22:17.784 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.784 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0A5.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0A5.completed"
Sep 22 23:22:17.784 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.784 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.785 DEBG [0] It's time to notify for 186
Sep 22 23:22:17.785 INFO Completion from [0] id:186 status:true
Sep 22 23:22:17.785 INFO [187/752] Repair commands completed
Sep 22 23:22:17.785 INFO Pop front: ReconcileIO { id: ReconciliationId(187), op: ExtentReopen { repair_id: ReconciliationId(187), extent_id: 165 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.785 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.785 INFO [0] received reconcile message
Sep 22 23:22:17.785 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(187), op: ExtentReopen { repair_id: ReconciliationId(187), extent_id: 165 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.785 INFO [0] client ExtentReopen { repair_id: ReconciliationId(187), extent_id: 165 }
Sep 22 23:22:17.785 INFO [1] received reconcile message
Sep 22 23:22:17.785 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(187), op: ExtentReopen { repair_id: ReconciliationId(187), extent_id: 165 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.785 INFO [1] client ExtentReopen { repair_id: ReconciliationId(187), extent_id: 165 }
Sep 22 23:22:17.785 INFO [2] received reconcile message
Sep 22 23:22:17.785 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(187), op: ExtentReopen { repair_id: ReconciliationId(187), extent_id: 165 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.785 INFO [2] client ExtentReopen { repair_id: ReconciliationId(187), extent_id: 165 }
Sep 22 23:22:17.785 DEBG 187 Reopen extent 165
Sep 22 23:22:17.786 DEBG 187 Reopen extent 165
Sep 22 23:22:17.786 DEBG 187 Reopen extent 165
Sep 22 23:22:17.787 DEBG [2] It's time to notify for 187
Sep 22 23:22:17.787 INFO Completion from [2] id:187 status:true
Sep 22 23:22:17.787 INFO [188/752] Repair commands completed
Sep 22 23:22:17.787 INFO Pop front: ReconcileIO { id: ReconciliationId(188), op: ExtentFlush { repair_id: ReconciliationId(188), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.787 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.787 INFO [0] received reconcile message
Sep 22 23:22:17.787 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(188), op: ExtentFlush { repair_id: ReconciliationId(188), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.787 INFO [0] client ExtentFlush { repair_id: ReconciliationId(188), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.787 INFO [1] received reconcile message
Sep 22 23:22:17.787 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(188), op: ExtentFlush { repair_id: ReconciliationId(188), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.787 INFO [1] client ExtentFlush { repair_id: ReconciliationId(188), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.787 INFO [2] received reconcile message
Sep 22 23:22:17.787 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(188), op: ExtentFlush { repair_id: ReconciliationId(188), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.787 INFO [2] client ExtentFlush { repair_id: ReconciliationId(188), extent_id: 118, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.787 DEBG 188 Flush extent 118 with f:2 g:2
Sep 22 23:22:17.787 DEBG Flush just extent 118 with f:2 and g:2
Sep 22 23:22:17.787 DEBG [1] It's time to notify for 188
Sep 22 23:22:17.787 INFO Completion from [1] id:188 status:true
Sep 22 23:22:17.787 INFO [189/752] Repair commands completed
Sep 22 23:22:17.787 INFO Pop front: ReconcileIO { id: ReconciliationId(189), op: ExtentClose { repair_id: ReconciliationId(189), extent_id: 118 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.787 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.787 INFO [0] received reconcile message
Sep 22 23:22:17.787 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(189), op: ExtentClose { repair_id: ReconciliationId(189), extent_id: 118 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.787 INFO [0] client ExtentClose { repair_id: ReconciliationId(189), extent_id: 118 }
Sep 22 23:22:17.787 INFO [1] received reconcile message
Sep 22 23:22:17.788 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(189), op: ExtentClose { repair_id: ReconciliationId(189), extent_id: 118 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.788 INFO [1] client ExtentClose { repair_id: ReconciliationId(189), extent_id: 118 }
Sep 22 23:22:17.788 INFO [2] received reconcile message
Sep 22 23:22:17.788 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(189), op: ExtentClose { repair_id: ReconciliationId(189), extent_id: 118 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.788 INFO [2] client ExtentClose { repair_id: ReconciliationId(189), extent_id: 118 }
Sep 22 23:22:17.788 DEBG 189 Close extent 118
Sep 22 23:22:17.788 DEBG 189 Close extent 118
Sep 22 23:22:17.788 DEBG 189 Close extent 118
Sep 22 23:22:17.789 DEBG [2] It's time to notify for 189
Sep 22 23:22:17.789 INFO Completion from [2] id:189 status:true
Sep 22 23:22:17.789 INFO [190/752] Repair commands completed
Sep 22 23:22:17.789 INFO Pop front: ReconcileIO { id: ReconciliationId(190), op: ExtentRepair { repair_id: ReconciliationId(190), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.789 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.789 INFO [0] received reconcile message
Sep 22 23:22:17.789 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(190), op: ExtentRepair { repair_id: ReconciliationId(190), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.789 INFO [0] client ExtentRepair { repair_id: ReconciliationId(190), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.789 INFO [0] Sending repair request ReconciliationId(190)
Sep 22 23:22:17.789 INFO [1] received reconcile message
Sep 22 23:22:17.789 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(190), op: ExtentRepair { repair_id: ReconciliationId(190), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.789 INFO [1] client ExtentRepair { repair_id: ReconciliationId(190), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.789 INFO [1] No action required ReconciliationId(190)
Sep 22 23:22:17.789 INFO [2] received reconcile message
Sep 22 23:22:17.789 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(190), op: ExtentRepair { repair_id: ReconciliationId(190), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.789 INFO [2] client ExtentRepair { repair_id: ReconciliationId(190), extent_id: 118, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.789 INFO [2] No action required ReconciliationId(190)
Sep 22 23:22:17.789 DEBG 190 Repair extent 118 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.789 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/076.copy"
Sep 22 23:22:17.813 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:17.816 DEBG Flush     :1078 extent_limit None deps:[JobId(1077), JobId(1076)] res:true f:28 g:1
Sep 22 23:22:17.816 INFO [lossy] sleeping 1 second
Sep 22 23:22:17.822 DEBG Read      :1077 deps:[JobId(1076)] res:true
Sep 22 23:22:17.852 INFO accepted connection, remote_addr: 127.0.0.1:42860, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.852 TRCE incoming request, uri: /extent/118/files, method: GET, req_id: 64194861-c875-4a7c-b46b-29e81635475a, remote_addr: 127.0.0.1:42860, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.852 INFO request completed, latency_us: 239, response_code: 200, uri: /extent/118/files, method: GET, req_id: 64194861-c875-4a7c-b46b-29e81635475a, remote_addr: 127.0.0.1:42860, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.853 INFO eid:118 Found repair files: ["076", "076.db"]
Sep 22 23:22:17.853 TRCE incoming request, uri: /newextent/118/data, method: GET, req_id: 6571e296-94b5-4a8e-aa64-d74774216b87, remote_addr: 127.0.0.1:42860, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.853 INFO request completed, latency_us: 322, response_code: 200, uri: /newextent/118/data, method: GET, req_id: 6571e296-94b5-4a8e-aa64-d74774216b87, remote_addr: 127.0.0.1:42860, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.858 TRCE incoming request, uri: /newextent/118/db, method: GET, req_id: c5de7095-c9fe-4212-948c-6f3ddc4c74bf, remote_addr: 127.0.0.1:42860, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.859 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/118/db, method: GET, req_id: c5de7095-c9fe-4212-948c-6f3ddc4c74bf, remote_addr: 127.0.0.1:42860, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.860 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/076.copy" to "/tmp/downstairs-zrMnlo6G/00/000/076.replace"
Sep 22 23:22:17.860 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.861 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/076.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.861 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/076"
Sep 22 23:22:17.861 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/076.db"
Sep 22 23:22:17.861 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.861 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/076.replace" to "/tmp/downstairs-zrMnlo6G/00/000/076.completed"
Sep 22 23:22:17.861 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.861 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.861 DEBG [0] It's time to notify for 190
Sep 22 23:22:17.862 INFO Completion from [0] id:190 status:true
Sep 22 23:22:17.862 INFO [191/752] Repair commands completed
Sep 22 23:22:17.862 INFO Pop front: ReconcileIO { id: ReconciliationId(191), op: ExtentReopen { repair_id: ReconciliationId(191), extent_id: 118 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.862 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.862 INFO [0] received reconcile message
Sep 22 23:22:17.862 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(191), op: ExtentReopen { repair_id: ReconciliationId(191), extent_id: 118 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.862 INFO [0] client ExtentReopen { repair_id: ReconciliationId(191), extent_id: 118 }
Sep 22 23:22:17.862 INFO [1] received reconcile message
Sep 22 23:22:17.862 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(191), op: ExtentReopen { repair_id: ReconciliationId(191), extent_id: 118 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.862 INFO [1] client ExtentReopen { repair_id: ReconciliationId(191), extent_id: 118 }
Sep 22 23:22:17.862 INFO [2] received reconcile message
Sep 22 23:22:17.862 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(191), op: ExtentReopen { repair_id: ReconciliationId(191), extent_id: 118 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.862 INFO [2] client ExtentReopen { repair_id: ReconciliationId(191), extent_id: 118 }
Sep 22 23:22:17.862 DEBG 191 Reopen extent 118
Sep 22 23:22:17.863 DEBG 191 Reopen extent 118
Sep 22 23:22:17.863 DEBG 191 Reopen extent 118
Sep 22 23:22:17.864 DEBG [2] It's time to notify for 191
Sep 22 23:22:17.864 INFO Completion from [2] id:191 status:true
Sep 22 23:22:17.864 INFO [192/752] Repair commands completed
Sep 22 23:22:17.864 INFO Pop front: ReconcileIO { id: ReconciliationId(192), op: ExtentFlush { repair_id: ReconciliationId(192), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.864 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.864 INFO [0] received reconcile message
Sep 22 23:22:17.864 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(192), op: ExtentFlush { repair_id: ReconciliationId(192), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.864 INFO [0] client ExtentFlush { repair_id: ReconciliationId(192), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.864 INFO [1] received reconcile message
Sep 22 23:22:17.864 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(192), op: ExtentFlush { repair_id: ReconciliationId(192), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.864 INFO [1] client ExtentFlush { repair_id: ReconciliationId(192), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.864 INFO [2] received reconcile message
Sep 22 23:22:17.864 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(192), op: ExtentFlush { repair_id: ReconciliationId(192), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.864 INFO [2] client ExtentFlush { repair_id: ReconciliationId(192), extent_id: 111, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.864 DEBG 192 Flush extent 111 with f:2 g:2
Sep 22 23:22:17.864 DEBG Flush just extent 111 with f:2 and g:2
Sep 22 23:22:17.864 DEBG [1] It's time to notify for 192
Sep 22 23:22:17.864 INFO Completion from [1] id:192 status:true
Sep 22 23:22:17.864 INFO [193/752] Repair commands completed
Sep 22 23:22:17.864 INFO Pop front: ReconcileIO { id: ReconciliationId(193), op: ExtentClose { repair_id: ReconciliationId(193), extent_id: 111 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.864 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.865 INFO [0] received reconcile message
Sep 22 23:22:17.865 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(193), op: ExtentClose { repair_id: ReconciliationId(193), extent_id: 111 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.865 INFO [0] client ExtentClose { repair_id: ReconciliationId(193), extent_id: 111 }
Sep 22 23:22:17.865 INFO [1] received reconcile message
Sep 22 23:22:17.865 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(193), op: ExtentClose { repair_id: ReconciliationId(193), extent_id: 111 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.865 INFO [1] client ExtentClose { repair_id: ReconciliationId(193), extent_id: 111 }
Sep 22 23:22:17.865 INFO [2] received reconcile message
Sep 22 23:22:17.865 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(193), op: ExtentClose { repair_id: ReconciliationId(193), extent_id: 111 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.865 INFO [2] client ExtentClose { repair_id: ReconciliationId(193), extent_id: 111 }
Sep 22 23:22:17.865 DEBG 193 Close extent 111
Sep 22 23:22:17.865 DEBG 193 Close extent 111
Sep 22 23:22:17.865 DEBG 193 Close extent 111
Sep 22 23:22:17.866 DEBG [2] It's time to notify for 193
Sep 22 23:22:17.866 INFO Completion from [2] id:193 status:true
Sep 22 23:22:17.866 INFO [194/752] Repair commands completed
Sep 22 23:22:17.866 INFO Pop front: ReconcileIO { id: ReconciliationId(194), op: ExtentRepair { repair_id: ReconciliationId(194), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.866 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.866 INFO [0] received reconcile message
Sep 22 23:22:17.866 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(194), op: ExtentRepair { repair_id: ReconciliationId(194), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.866 INFO [0] client ExtentRepair { repair_id: ReconciliationId(194), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.866 INFO [0] Sending repair request ReconciliationId(194)
Sep 22 23:22:17.866 INFO [1] received reconcile message
Sep 22 23:22:17.866 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(194), op: ExtentRepair { repair_id: ReconciliationId(194), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.866 INFO [1] client ExtentRepair { repair_id: ReconciliationId(194), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.866 INFO [1] No action required ReconciliationId(194)
Sep 22 23:22:17.866 INFO [2] received reconcile message
Sep 22 23:22:17.866 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(194), op: ExtentRepair { repair_id: ReconciliationId(194), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.866 INFO [2] client ExtentRepair { repair_id: ReconciliationId(194), extent_id: 111, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.866 INFO [2] No action required ReconciliationId(194)
Sep 22 23:22:17.866 DEBG 194 Repair extent 111 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.866 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/06F.copy"
Sep 22 23:22:17.891 DEBG IO Read  1079 has deps [JobId(1078)]
Sep 22 23:22:17.929 INFO accepted connection, remote_addr: 127.0.0.1:43732, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.929 TRCE incoming request, uri: /extent/111/files, method: GET, req_id: 61ab86c4-55c1-4d1f-b6fe-a9e9ef6e110e, remote_addr: 127.0.0.1:43732, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.930 INFO request completed, latency_us: 197, response_code: 200, uri: /extent/111/files, method: GET, req_id: 61ab86c4-55c1-4d1f-b6fe-a9e9ef6e110e, remote_addr: 127.0.0.1:43732, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.930 INFO eid:111 Found repair files: ["06F", "06F.db"]
Sep 22 23:22:17.930 TRCE incoming request, uri: /newextent/111/data, method: GET, req_id: 8eab960c-dbf2-4ad6-b86b-2449d4f4740d, remote_addr: 127.0.0.1:43732, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.930 INFO request completed, latency_us: 326, response_code: 200, uri: /newextent/111/data, method: GET, req_id: 8eab960c-dbf2-4ad6-b86b-2449d4f4740d, remote_addr: 127.0.0.1:43732, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.935 TRCE incoming request, uri: /newextent/111/db, method: GET, req_id: f6dd1e53-8fcc-4d09-9fd0-e53082f98423, remote_addr: 127.0.0.1:43732, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.936 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/111/db, method: GET, req_id: f6dd1e53-8fcc-4d09-9fd0-e53082f98423, remote_addr: 127.0.0.1:43732, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:17.937 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/06F.copy" to "/tmp/downstairs-zrMnlo6G/00/000/06F.replace"
Sep 22 23:22:17.937 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.938 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/06F.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.938 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06F"
Sep 22 23:22:17.938 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06F.db"
Sep 22 23:22:17.938 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.938 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/06F.replace" to "/tmp/downstairs-zrMnlo6G/00/000/06F.completed"
Sep 22 23:22:17.938 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.938 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:17.938 DEBG [0] It's time to notify for 194
Sep 22 23:22:17.938 INFO Completion from [0] id:194 status:true
Sep 22 23:22:17.938 INFO [195/752] Repair commands completed
Sep 22 23:22:17.938 INFO Pop front: ReconcileIO { id: ReconciliationId(195), op: ExtentReopen { repair_id: ReconciliationId(195), extent_id: 111 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.938 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.938 INFO [0] received reconcile message
Sep 22 23:22:17.938 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(195), op: ExtentReopen { repair_id: ReconciliationId(195), extent_id: 111 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.938 INFO [0] client ExtentReopen { repair_id: ReconciliationId(195), extent_id: 111 }
Sep 22 23:22:17.939 INFO [1] received reconcile message
Sep 22 23:22:17.939 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(195), op: ExtentReopen { repair_id: ReconciliationId(195), extent_id: 111 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.939 INFO [1] client ExtentReopen { repair_id: ReconciliationId(195), extent_id: 111 }
Sep 22 23:22:17.939 INFO [2] received reconcile message
Sep 22 23:22:17.939 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(195), op: ExtentReopen { repair_id: ReconciliationId(195), extent_id: 111 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.939 INFO [2] client ExtentReopen { repair_id: ReconciliationId(195), extent_id: 111 }
Sep 22 23:22:17.939 DEBG 195 Reopen extent 111
Sep 22 23:22:17.939 DEBG 195 Reopen extent 111
Sep 22 23:22:17.940 DEBG 195 Reopen extent 111
Sep 22 23:22:17.940 DEBG [2] It's time to notify for 195
Sep 22 23:22:17.940 INFO Completion from [2] id:195 status:true
Sep 22 23:22:17.940 INFO [196/752] Repair commands completed
Sep 22 23:22:17.940 INFO Pop front: ReconcileIO { id: ReconciliationId(196), op: ExtentFlush { repair_id: ReconciliationId(196), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.940 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.941 INFO [0] received reconcile message
Sep 22 23:22:17.941 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(196), op: ExtentFlush { repair_id: ReconciliationId(196), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.941 INFO [0] client ExtentFlush { repair_id: ReconciliationId(196), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.941 INFO [1] received reconcile message
Sep 22 23:22:17.941 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(196), op: ExtentFlush { repair_id: ReconciliationId(196), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:17.941 INFO [1] client ExtentFlush { repair_id: ReconciliationId(196), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.941 INFO [2] received reconcile message
Sep 22 23:22:17.941 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(196), op: ExtentFlush { repair_id: ReconciliationId(196), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.941 INFO [2] client ExtentFlush { repair_id: ReconciliationId(196), extent_id: 74, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:17.941 DEBG 196 Flush extent 74 with f:2 g:2
Sep 22 23:22:17.941 DEBG Flush just extent 74 with f:2 and g:2
Sep 22 23:22:17.941 DEBG [1] It's time to notify for 196
Sep 22 23:22:17.941 INFO Completion from [1] id:196 status:true
Sep 22 23:22:17.941 INFO [197/752] Repair commands completed
Sep 22 23:22:17.941 INFO Pop front: ReconcileIO { id: ReconciliationId(197), op: ExtentClose { repair_id: ReconciliationId(197), extent_id: 74 }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.941 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.941 INFO [0] received reconcile message
Sep 22 23:22:17.941 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(197), op: ExtentClose { repair_id: ReconciliationId(197), extent_id: 74 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.941 INFO [0] client ExtentClose { repair_id: ReconciliationId(197), extent_id: 74 }
Sep 22 23:22:17.941 INFO [1] received reconcile message
Sep 22 23:22:17.941 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(197), op: ExtentClose { repair_id: ReconciliationId(197), extent_id: 74 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.941 INFO [1] client ExtentClose { repair_id: ReconciliationId(197), extent_id: 74 }
Sep 22 23:22:17.941 INFO [2] received reconcile message
Sep 22 23:22:17.941 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(197), op: ExtentClose { repair_id: ReconciliationId(197), extent_id: 74 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:17.941 INFO [2] client ExtentClose { repair_id: ReconciliationId(197), extent_id: 74 }
Sep 22 23:22:17.941 DEBG 197 Close extent 74
Sep 22 23:22:17.942 DEBG 197 Close extent 74
Sep 22 23:22:17.942 DEBG 197 Close extent 74
Sep 22 23:22:17.942 DEBG [2] It's time to notify for 197
Sep 22 23:22:17.942 INFO Completion from [2] id:197 status:true
Sep 22 23:22:17.942 INFO [198/752] Repair commands completed
Sep 22 23:22:17.942 INFO Pop front: ReconcileIO { id: ReconciliationId(198), op: ExtentRepair { repair_id: ReconciliationId(198), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:17.942 INFO Sent repair work, now wait for resp
Sep 22 23:22:17.942 INFO [0] received reconcile message
Sep 22 23:22:17.942 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(198), op: ExtentRepair { repair_id: ReconciliationId(198), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:17.942 INFO [0] client ExtentRepair { repair_id: ReconciliationId(198), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.942 INFO [0] Sending repair request ReconciliationId(198)
Sep 22 23:22:17.943 INFO [1] received reconcile message
Sep 22 23:22:17.943 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(198), op: ExtentRepair { repair_id: ReconciliationId(198), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:17.943 INFO [1] client ExtentRepair { repair_id: ReconciliationId(198), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.943 INFO [1] No action required ReconciliationId(198)
Sep 22 23:22:17.943 INFO [2] received reconcile message
Sep 22 23:22:17.943 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(198), op: ExtentRepair { repair_id: ReconciliationId(198), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:17.943 INFO [2] client ExtentRepair { repair_id: ReconciliationId(198), extent_id: 74, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:17.943 INFO [2] No action required ReconciliationId(198)
Sep 22 23:22:17.943 DEBG 198 Repair extent 74 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:17.943 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/04A.copy"
Sep 22 23:22:18.008 INFO accepted connection, remote_addr: 127.0.0.1:37752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.008 TRCE incoming request, uri: /extent/74/files, method: GET, req_id: 64c89e0a-fbba-4d06-89f1-aacd6404aad5, remote_addr: 127.0.0.1:37752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.009 INFO request completed, latency_us: 268, response_code: 200, uri: /extent/74/files, method: GET, req_id: 64c89e0a-fbba-4d06-89f1-aacd6404aad5, remote_addr: 127.0.0.1:37752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.009 INFO eid:74 Found repair files: ["04A", "04A.db"]
Sep 22 23:22:18.009 TRCE incoming request, uri: /newextent/74/data, method: GET, req_id: b7ad28f1-31fa-43e7-abf9-df8bfcbf3709, remote_addr: 127.0.0.1:37752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.010 INFO request completed, latency_us: 361, response_code: 200, uri: /newextent/74/data, method: GET, req_id: b7ad28f1-31fa-43e7-abf9-df8bfcbf3709, remote_addr: 127.0.0.1:37752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.015 TRCE incoming request, uri: /newextent/74/db, method: GET, req_id: a8fe68c6-22ca-45d6-b791-86fb2037dd26, remote_addr: 127.0.0.1:37752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.015 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/74/db, method: GET, req_id: a8fe68c6-22ca-45d6-b791-86fb2037dd26, remote_addr: 127.0.0.1:37752, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.016 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/04A.copy" to "/tmp/downstairs-zrMnlo6G/00/000/04A.replace"
Sep 22 23:22:18.016 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.017 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/04A.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.018 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04A"
Sep 22 23:22:18.018 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04A.db"
Sep 22 23:22:18.018 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.018 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/04A.replace" to "/tmp/downstairs-zrMnlo6G/00/000/04A.completed"
Sep 22 23:22:18.018 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.018 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.018 DEBG [0] It's time to notify for 198
Sep 22 23:22:18.018 INFO Completion from [0] id:198 status:true
Sep 22 23:22:18.018 INFO [199/752] Repair commands completed
Sep 22 23:22:18.018 INFO Pop front: ReconcileIO { id: ReconciliationId(199), op: ExtentReopen { repair_id: ReconciliationId(199), extent_id: 74 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.018 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.018 INFO [0] received reconcile message
Sep 22 23:22:18.018 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(199), op: ExtentReopen { repair_id: ReconciliationId(199), extent_id: 74 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.018 INFO [0] client ExtentReopen { repair_id: ReconciliationId(199), extent_id: 74 }
Sep 22 23:22:18.018 INFO [1] received reconcile message
Sep 22 23:22:18.018 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(199), op: ExtentReopen { repair_id: ReconciliationId(199), extent_id: 74 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.018 INFO [1] client ExtentReopen { repair_id: ReconciliationId(199), extent_id: 74 }
Sep 22 23:22:18.018 INFO [2] received reconcile message
Sep 22 23:22:18.018 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(199), op: ExtentReopen { repair_id: ReconciliationId(199), extent_id: 74 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.018 INFO [2] client ExtentReopen { repair_id: ReconciliationId(199), extent_id: 74 }
Sep 22 23:22:18.019 DEBG 199 Reopen extent 74
Sep 22 23:22:18.019 DEBG 199 Reopen extent 74
Sep 22 23:22:18.020 DEBG 199 Reopen extent 74
Sep 22 23:22:18.021 DEBG [2] It's time to notify for 199
Sep 22 23:22:18.021 INFO Completion from [2] id:199 status:true
Sep 22 23:22:18.021 INFO [200/752] Repair commands completed
Sep 22 23:22:18.021 INFO Pop front: ReconcileIO { id: ReconciliationId(200), op: ExtentFlush { repair_id: ReconciliationId(200), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.021 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.021 INFO [0] received reconcile message
Sep 22 23:22:18.021 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(200), op: ExtentFlush { repair_id: ReconciliationId(200), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.021 INFO [0] client ExtentFlush { repair_id: ReconciliationId(200), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.021 INFO [1] received reconcile message
Sep 22 23:22:18.021 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(200), op: ExtentFlush { repair_id: ReconciliationId(200), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.021 INFO [1] client ExtentFlush { repair_id: ReconciliationId(200), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.021 INFO [2] received reconcile message
Sep 22 23:22:18.021 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(200), op: ExtentFlush { repair_id: ReconciliationId(200), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.021 INFO [2] client ExtentFlush { repair_id: ReconciliationId(200), extent_id: 117, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.021 DEBG 200 Flush extent 117 with f:2 g:2
Sep 22 23:22:18.021 DEBG Flush just extent 117 with f:2 and g:2
Sep 22 23:22:18.021 DEBG [1] It's time to notify for 200
Sep 22 23:22:18.021 INFO Completion from [1] id:200 status:true
Sep 22 23:22:18.021 INFO [201/752] Repair commands completed
Sep 22 23:22:18.021 INFO Pop front: ReconcileIO { id: ReconciliationId(201), op: ExtentClose { repair_id: ReconciliationId(201), extent_id: 117 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.021 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.021 INFO [0] received reconcile message
Sep 22 23:22:18.021 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(201), op: ExtentClose { repair_id: ReconciliationId(201), extent_id: 117 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.021 INFO [0] client ExtentClose { repair_id: ReconciliationId(201), extent_id: 117 }
Sep 22 23:22:18.021 INFO [1] received reconcile message
Sep 22 23:22:18.021 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(201), op: ExtentClose { repair_id: ReconciliationId(201), extent_id: 117 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.021 INFO [1] client ExtentClose { repair_id: ReconciliationId(201), extent_id: 117 }
Sep 22 23:22:18.021 INFO [2] received reconcile message
Sep 22 23:22:18.021 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(201), op: ExtentClose { repair_id: ReconciliationId(201), extent_id: 117 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.021 INFO [2] client ExtentClose { repair_id: ReconciliationId(201), extent_id: 117 }
Sep 22 23:22:18.022 DEBG 201 Close extent 117
Sep 22 23:22:18.022 DEBG 201 Close extent 117
Sep 22 23:22:18.022 DEBG 201 Close extent 117
Sep 22 23:22:18.022 DEBG [2] It's time to notify for 201
Sep 22 23:22:18.023 INFO Completion from [2] id:201 status:true
Sep 22 23:22:18.023 INFO [202/752] Repair commands completed
Sep 22 23:22:18.023 INFO Pop front: ReconcileIO { id: ReconciliationId(202), op: ExtentRepair { repair_id: ReconciliationId(202), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.023 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.023 INFO [0] received reconcile message
Sep 22 23:22:18.023 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(202), op: ExtentRepair { repair_id: ReconciliationId(202), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.023 INFO [0] client ExtentRepair { repair_id: ReconciliationId(202), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.023 INFO [0] Sending repair request ReconciliationId(202)
Sep 22 23:22:18.023 INFO [1] received reconcile message
Sep 22 23:22:18.023 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(202), op: ExtentRepair { repair_id: ReconciliationId(202), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.023 INFO [1] client ExtentRepair { repair_id: ReconciliationId(202), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.023 INFO [1] No action required ReconciliationId(202)
Sep 22 23:22:18.023 INFO [2] received reconcile message
Sep 22 23:22:18.023 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(202), op: ExtentRepair { repair_id: ReconciliationId(202), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.023 INFO [2] client ExtentRepair { repair_id: ReconciliationId(202), extent_id: 117, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.023 INFO [2] No action required ReconciliationId(202)
Sep 22 23:22:18.023 DEBG 202 Repair extent 117 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.023 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/075.copy"
Sep 22 23:22:18.088 INFO accepted connection, remote_addr: 127.0.0.1:63962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.088 TRCE incoming request, uri: /extent/117/files, method: GET, req_id: 63f21dd0-f05b-4eee-a36c-8dfababa19a0, remote_addr: 127.0.0.1:63962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.088 INFO request completed, latency_us: 200, response_code: 200, uri: /extent/117/files, method: GET, req_id: 63f21dd0-f05b-4eee-a36c-8dfababa19a0, remote_addr: 127.0.0.1:63962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.089 INFO eid:117 Found repair files: ["075", "075.db"]
Sep 22 23:22:18.089 TRCE incoming request, uri: /newextent/117/data, method: GET, req_id: 3fa22972-afc6-4a73-89df-f022914ea3e3, remote_addr: 127.0.0.1:63962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.089 INFO request completed, latency_us: 251, response_code: 200, uri: /newextent/117/data, method: GET, req_id: 3fa22972-afc6-4a73-89df-f022914ea3e3, remote_addr: 127.0.0.1:63962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.094 TRCE incoming request, uri: /newextent/117/db, method: GET, req_id: 82ac3232-8296-4d8b-9763-879aa10a7c10, remote_addr: 127.0.0.1:63962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.094 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/117/db, method: GET, req_id: 82ac3232-8296-4d8b-9763-879aa10a7c10, remote_addr: 127.0.0.1:63962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.096 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/075.copy" to "/tmp/downstairs-zrMnlo6G/00/000/075.replace"
Sep 22 23:22:18.096 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.096 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/075.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.097 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/075"
Sep 22 23:22:18.097 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/075.db"
Sep 22 23:22:18.097 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.097 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/075.replace" to "/tmp/downstairs-zrMnlo6G/00/000/075.completed"
Sep 22 23:22:18.097 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.097 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.097 DEBG [0] It's time to notify for 202
Sep 22 23:22:18.097 INFO Completion from [0] id:202 status:true
Sep 22 23:22:18.097 INFO [203/752] Repair commands completed
Sep 22 23:22:18.097 INFO Pop front: ReconcileIO { id: ReconciliationId(203), op: ExtentReopen { repair_id: ReconciliationId(203), extent_id: 117 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.097 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.097 INFO [0] received reconcile message
Sep 22 23:22:18.097 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(203), op: ExtentReopen { repair_id: ReconciliationId(203), extent_id: 117 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.097 INFO [0] client ExtentReopen { repair_id: ReconciliationId(203), extent_id: 117 }
Sep 22 23:22:18.097 INFO [1] received reconcile message
Sep 22 23:22:18.097 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(203), op: ExtentReopen { repair_id: ReconciliationId(203), extent_id: 117 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.097 INFO [1] client ExtentReopen { repair_id: ReconciliationId(203), extent_id: 117 }
Sep 22 23:22:18.097 INFO [2] received reconcile message
Sep 22 23:22:18.097 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(203), op: ExtentReopen { repair_id: ReconciliationId(203), extent_id: 117 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.098 INFO [2] client ExtentReopen { repair_id: ReconciliationId(203), extent_id: 117 }
Sep 22 23:22:18.098 DEBG 203 Reopen extent 117
Sep 22 23:22:18.098 DEBG 203 Reopen extent 117
Sep 22 23:22:18.099 DEBG 203 Reopen extent 117
Sep 22 23:22:18.099 DEBG [2] It's time to notify for 203
Sep 22 23:22:18.099 INFO Completion from [2] id:203 status:true
Sep 22 23:22:18.099 INFO [204/752] Repair commands completed
Sep 22 23:22:18.099 INFO Pop front: ReconcileIO { id: ReconciliationId(204), op: ExtentFlush { repair_id: ReconciliationId(204), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.099 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.099 INFO [0] received reconcile message
Sep 22 23:22:18.099 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(204), op: ExtentFlush { repair_id: ReconciliationId(204), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.099 INFO [0] client ExtentFlush { repair_id: ReconciliationId(204), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.099 INFO [1] received reconcile message
Sep 22 23:22:18.100 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(204), op: ExtentFlush { repair_id: ReconciliationId(204), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.100 INFO [1] client ExtentFlush { repair_id: ReconciliationId(204), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.100 INFO [2] received reconcile message
Sep 22 23:22:18.100 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(204), op: ExtentFlush { repair_id: ReconciliationId(204), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.100 INFO [2] client ExtentFlush { repair_id: ReconciliationId(204), extent_id: 75, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.100 DEBG 204 Flush extent 75 with f:2 g:2
Sep 22 23:22:18.100 DEBG Flush just extent 75 with f:2 and g:2
Sep 22 23:22:18.100 DEBG [1] It's time to notify for 204
Sep 22 23:22:18.100 INFO Completion from [1] id:204 status:true
Sep 22 23:22:18.100 INFO [205/752] Repair commands completed
Sep 22 23:22:18.100 INFO Pop front: ReconcileIO { id: ReconciliationId(205), op: ExtentClose { repair_id: ReconciliationId(205), extent_id: 75 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.100 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.100 INFO [0] received reconcile message
Sep 22 23:22:18.100 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(205), op: ExtentClose { repair_id: ReconciliationId(205), extent_id: 75 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.100 INFO [0] client ExtentClose { repair_id: ReconciliationId(205), extent_id: 75 }
Sep 22 23:22:18.100 INFO [1] received reconcile message
Sep 22 23:22:18.100 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(205), op: ExtentClose { repair_id: ReconciliationId(205), extent_id: 75 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.100 INFO [1] client ExtentClose { repair_id: ReconciliationId(205), extent_id: 75 }
Sep 22 23:22:18.100 INFO [2] received reconcile message
Sep 22 23:22:18.100 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(205), op: ExtentClose { repair_id: ReconciliationId(205), extent_id: 75 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.100 INFO [2] client ExtentClose { repair_id: ReconciliationId(205), extent_id: 75 }
Sep 22 23:22:18.100 DEBG 205 Close extent 75
Sep 22 23:22:18.101 DEBG 205 Close extent 75
Sep 22 23:22:18.101 DEBG 205 Close extent 75
Sep 22 23:22:18.101 DEBG [2] It's time to notify for 205
Sep 22 23:22:18.101 INFO Completion from [2] id:205 status:true
Sep 22 23:22:18.101 INFO [206/752] Repair commands completed
Sep 22 23:22:18.101 INFO Pop front: ReconcileIO { id: ReconciliationId(206), op: ExtentRepair { repair_id: ReconciliationId(206), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.101 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.101 INFO [0] received reconcile message
Sep 22 23:22:18.101 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(206), op: ExtentRepair { repair_id: ReconciliationId(206), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.101 INFO [0] client ExtentRepair { repair_id: ReconciliationId(206), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.101 INFO [0] Sending repair request ReconciliationId(206)
Sep 22 23:22:18.101 INFO [1] received reconcile message
Sep 22 23:22:18.101 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(206), op: ExtentRepair { repair_id: ReconciliationId(206), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.102 INFO [1] client ExtentRepair { repair_id: ReconciliationId(206), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.102 INFO [1] No action required ReconciliationId(206)
Sep 22 23:22:18.102 INFO [2] received reconcile message
Sep 22 23:22:18.102 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(206), op: ExtentRepair { repair_id: ReconciliationId(206), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.102 INFO [2] client ExtentRepair { repair_id: ReconciliationId(206), extent_id: 75, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.102 INFO [2] No action required ReconciliationId(206)
Sep 22 23:22:18.102 DEBG 206 Repair extent 75 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.102 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/04B.copy"
Sep 22 23:22:18.166 INFO accepted connection, remote_addr: 127.0.0.1:43026, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.166 TRCE incoming request, uri: /extent/75/files, method: GET, req_id: aead01ee-3d45-4dad-96aa-dad051404bf8, remote_addr: 127.0.0.1:43026, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.167 INFO request completed, latency_us: 217, response_code: 200, uri: /extent/75/files, method: GET, req_id: aead01ee-3d45-4dad-96aa-dad051404bf8, remote_addr: 127.0.0.1:43026, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.167 INFO eid:75 Found repair files: ["04B", "04B.db"]
Sep 22 23:22:18.167 TRCE incoming request, uri: /newextent/75/data, method: GET, req_id: 487e4630-5e41-48c9-bdad-584ebb7255c6, remote_addr: 127.0.0.1:43026, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.168 INFO request completed, latency_us: 312, response_code: 200, uri: /newextent/75/data, method: GET, req_id: 487e4630-5e41-48c9-bdad-584ebb7255c6, remote_addr: 127.0.0.1:43026, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.173 TRCE incoming request, uri: /newextent/75/db, method: GET, req_id: 83bb672c-9d8d-4646-9468-52e74e7da250, remote_addr: 127.0.0.1:43026, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.173 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/75/db, method: GET, req_id: 83bb672c-9d8d-4646-9468-52e74e7da250, remote_addr: 127.0.0.1:43026, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.174 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/04B.copy" to "/tmp/downstairs-zrMnlo6G/00/000/04B.replace"
Sep 22 23:22:18.174 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.175 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/04B.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.175 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04B"
Sep 22 23:22:18.175 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04B.db"
Sep 22 23:22:18.175 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.175 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/04B.replace" to "/tmp/downstairs-zrMnlo6G/00/000/04B.completed"
Sep 22 23:22:18.175 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.175 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.176 DEBG [0] It's time to notify for 206
Sep 22 23:22:18.176 INFO Completion from [0] id:206 status:true
Sep 22 23:22:18.176 INFO [207/752] Repair commands completed
Sep 22 23:22:18.176 INFO Pop front: ReconcileIO { id: ReconciliationId(207), op: ExtentReopen { repair_id: ReconciliationId(207), extent_id: 75 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.176 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.176 INFO [0] received reconcile message
Sep 22 23:22:18.176 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(207), op: ExtentReopen { repair_id: ReconciliationId(207), extent_id: 75 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.176 INFO [0] client ExtentReopen { repair_id: ReconciliationId(207), extent_id: 75 }
Sep 22 23:22:18.176 INFO [1] received reconcile message
Sep 22 23:22:18.176 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(207), op: ExtentReopen { repair_id: ReconciliationId(207), extent_id: 75 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.176 INFO [1] client ExtentReopen { repair_id: ReconciliationId(207), extent_id: 75 }
Sep 22 23:22:18.176 INFO [2] received reconcile message
Sep 22 23:22:18.176 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(207), op: ExtentReopen { repair_id: ReconciliationId(207), extent_id: 75 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.176 INFO [2] client ExtentReopen { repair_id: ReconciliationId(207), extent_id: 75 }
Sep 22 23:22:18.176 DEBG 207 Reopen extent 75
Sep 22 23:22:18.177 DEBG 207 Reopen extent 75
Sep 22 23:22:18.177 DEBG 207 Reopen extent 75
Sep 22 23:22:18.178 DEBG [2] It's time to notify for 207
Sep 22 23:22:18.178 INFO Completion from [2] id:207 status:true
Sep 22 23:22:18.178 INFO [208/752] Repair commands completed
Sep 22 23:22:18.178 INFO Pop front: ReconcileIO { id: ReconciliationId(208), op: ExtentFlush { repair_id: ReconciliationId(208), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.178 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.178 INFO [0] received reconcile message
Sep 22 23:22:18.178 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(208), op: ExtentFlush { repair_id: ReconciliationId(208), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.178 INFO [0] client ExtentFlush { repair_id: ReconciliationId(208), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.178 INFO [1] received reconcile message
Sep 22 23:22:18.178 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(208), op: ExtentFlush { repair_id: ReconciliationId(208), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.178 INFO [1] client ExtentFlush { repair_id: ReconciliationId(208), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.178 INFO [2] received reconcile message
Sep 22 23:22:18.178 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(208), op: ExtentFlush { repair_id: ReconciliationId(208), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.178 INFO [2] client ExtentFlush { repair_id: ReconciliationId(208), extent_id: 36, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.178 DEBG 208 Flush extent 36 with f:2 g:2
Sep 22 23:22:18.178 DEBG Flush just extent 36 with f:2 and g:2
Sep 22 23:22:18.178 DEBG [1] It's time to notify for 208
Sep 22 23:22:18.178 INFO Completion from [1] id:208 status:true
Sep 22 23:22:18.178 INFO [209/752] Repair commands completed
Sep 22 23:22:18.178 INFO Pop front: ReconcileIO { id: ReconciliationId(209), op: ExtentClose { repair_id: ReconciliationId(209), extent_id: 36 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.178 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.179 INFO [0] received reconcile message
Sep 22 23:22:18.179 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(209), op: ExtentClose { repair_id: ReconciliationId(209), extent_id: 36 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.179 INFO [0] client ExtentClose { repair_id: ReconciliationId(209), extent_id: 36 }
Sep 22 23:22:18.179 INFO [1] received reconcile message
Sep 22 23:22:18.179 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(209), op: ExtentClose { repair_id: ReconciliationId(209), extent_id: 36 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.179 INFO [1] client ExtentClose { repair_id: ReconciliationId(209), extent_id: 36 }
Sep 22 23:22:18.179 INFO [2] received reconcile message
Sep 22 23:22:18.179 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(209), op: ExtentClose { repair_id: ReconciliationId(209), extent_id: 36 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.179 INFO [2] client ExtentClose { repair_id: ReconciliationId(209), extent_id: 36 }
Sep 22 23:22:18.179 DEBG 209 Close extent 36
Sep 22 23:22:18.179 DEBG 209 Close extent 36
Sep 22 23:22:18.179 DEBG 209 Close extent 36
Sep 22 23:22:18.180 DEBG [2] It's time to notify for 209
Sep 22 23:22:18.180 INFO Completion from [2] id:209 status:true
Sep 22 23:22:18.180 INFO [210/752] Repair commands completed
Sep 22 23:22:18.180 INFO Pop front: ReconcileIO { id: ReconciliationId(210), op: ExtentRepair { repair_id: ReconciliationId(210), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.180 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.180 INFO [0] received reconcile message
Sep 22 23:22:18.180 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(210), op: ExtentRepair { repair_id: ReconciliationId(210), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.180 INFO [0] client ExtentRepair { repair_id: ReconciliationId(210), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.180 INFO [0] Sending repair request ReconciliationId(210)
Sep 22 23:22:18.180 INFO [1] received reconcile message
Sep 22 23:22:18.180 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(210), op: ExtentRepair { repair_id: ReconciliationId(210), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.180 INFO [1] client ExtentRepair { repair_id: ReconciliationId(210), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.180 INFO [1] No action required ReconciliationId(210)
Sep 22 23:22:18.180 INFO [2] received reconcile message
Sep 22 23:22:18.180 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(210), op: ExtentRepair { repair_id: ReconciliationId(210), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.180 INFO [2] client ExtentRepair { repair_id: ReconciliationId(210), extent_id: 36, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.180 INFO [2] No action required ReconciliationId(210)
Sep 22 23:22:18.180 DEBG 210 Repair extent 36 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.180 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/024.copy"
Sep 22 23:22:18.232 DEBG up_ds_listen was notified
Sep 22 23:22:18.232 DEBG up_ds_listen process 1078
Sep 22 23:22:18.232 DEBG [A] ack job 1078:79, : downstairs
Sep 22 23:22:18.232 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:18.241 WARN returning error on flush!
Sep 22 23:22:18.241 DEBG Flush     :1078 extent_limit None deps:[JobId(1077), JobId(1076)] res:false f:28 g:1
Sep 22 23:22:18.241 DEBG Flush     :1078 extent_limit None deps:[JobId(1077), JobId(1076)] res:true f:28 g:1
Sep 22 23:22:18.245 INFO accepted connection, remote_addr: 127.0.0.1:35803, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.245 TRCE incoming request, uri: /extent/36/files, method: GET, req_id: e88d4089-ac79-4993-96ac-f22fd70a018c, remote_addr: 127.0.0.1:35803, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.245 INFO request completed, latency_us: 189, response_code: 200, uri: /extent/36/files, method: GET, req_id: e88d4089-ac79-4993-96ac-f22fd70a018c, remote_addr: 127.0.0.1:35803, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.246 INFO eid:36 Found repair files: ["024", "024.db"]
Sep 22 23:22:18.246 TRCE incoming request, uri: /newextent/36/data, method: GET, req_id: 73106bb5-5ccd-4152-a90f-d1e25b896f8d, remote_addr: 127.0.0.1:35803, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.246 INFO request completed, latency_us: 246, response_code: 200, uri: /newextent/36/data, method: GET, req_id: 73106bb5-5ccd-4152-a90f-d1e25b896f8d, remote_addr: 127.0.0.1:35803, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.247 DEBG Read      :1079 deps:[JobId(1078)] res:true
Sep 22 23:22:18.251 TRCE incoming request, uri: /newextent/36/db, method: GET, req_id: 271d5338-c5fa-4a08-866d-9902d69d3324, remote_addr: 127.0.0.1:35803, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.252 INFO request completed, latency_us: 293, response_code: 200, uri: /newextent/36/db, method: GET, req_id: 271d5338-c5fa-4a08-866d-9902d69d3324, remote_addr: 127.0.0.1:35803, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.253 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/024.copy" to "/tmp/downstairs-zrMnlo6G/00/000/024.replace"
Sep 22 23:22:18.253 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.253 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/024.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.254 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/024"
Sep 22 23:22:18.254 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/024.db"
Sep 22 23:22:18.254 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.254 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/024.replace" to "/tmp/downstairs-zrMnlo6G/00/000/024.completed"
Sep 22 23:22:18.254 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.254 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.254 DEBG [0] It's time to notify for 210
Sep 22 23:22:18.254 INFO Completion from [0] id:210 status:true
Sep 22 23:22:18.254 INFO [211/752] Repair commands completed
Sep 22 23:22:18.254 INFO Pop front: ReconcileIO { id: ReconciliationId(211), op: ExtentReopen { repair_id: ReconciliationId(211), extent_id: 36 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.254 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.254 INFO [0] received reconcile message
Sep 22 23:22:18.254 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(211), op: ExtentReopen { repair_id: ReconciliationId(211), extent_id: 36 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.254 INFO [0] client ExtentReopen { repair_id: ReconciliationId(211), extent_id: 36 }
Sep 22 23:22:18.254 INFO [1] received reconcile message
Sep 22 23:22:18.254 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(211), op: ExtentReopen { repair_id: ReconciliationId(211), extent_id: 36 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.254 INFO [1] client ExtentReopen { repair_id: ReconciliationId(211), extent_id: 36 }
Sep 22 23:22:18.255 INFO [2] received reconcile message
Sep 22 23:22:18.255 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(211), op: ExtentReopen { repair_id: ReconciliationId(211), extent_id: 36 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.255 INFO [2] client ExtentReopen { repair_id: ReconciliationId(211), extent_id: 36 }
Sep 22 23:22:18.255 DEBG 211 Reopen extent 36
Sep 22 23:22:18.255 DEBG 211 Reopen extent 36
Sep 22 23:22:18.256 DEBG 211 Reopen extent 36
Sep 22 23:22:18.256 DEBG [2] It's time to notify for 211
Sep 22 23:22:18.256 INFO Completion from [2] id:211 status:true
Sep 22 23:22:18.256 INFO [212/752] Repair commands completed
Sep 22 23:22:18.256 INFO Pop front: ReconcileIO { id: ReconciliationId(212), op: ExtentFlush { repair_id: ReconciliationId(212), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.256 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.256 INFO [0] received reconcile message
Sep 22 23:22:18.256 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(212), op: ExtentFlush { repair_id: ReconciliationId(212), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.256 INFO [0] client ExtentFlush { repair_id: ReconciliationId(212), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.256 INFO [1] received reconcile message
Sep 22 23:22:18.257 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(212), op: ExtentFlush { repair_id: ReconciliationId(212), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.257 INFO [1] client ExtentFlush { repair_id: ReconciliationId(212), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.257 INFO [2] received reconcile message
Sep 22 23:22:18.257 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(212), op: ExtentFlush { repair_id: ReconciliationId(212), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.257 INFO [2] client ExtentFlush { repair_id: ReconciliationId(212), extent_id: 68, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.257 DEBG 212 Flush extent 68 with f:2 g:2
Sep 22 23:22:18.257 DEBG Flush just extent 68 with f:2 and g:2
Sep 22 23:22:18.257 DEBG [1] It's time to notify for 212
Sep 22 23:22:18.257 INFO Completion from [1] id:212 status:true
Sep 22 23:22:18.257 INFO [213/752] Repair commands completed
Sep 22 23:22:18.257 INFO Pop front: ReconcileIO { id: ReconciliationId(213), op: ExtentClose { repair_id: ReconciliationId(213), extent_id: 68 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.257 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.257 INFO [0] received reconcile message
Sep 22 23:22:18.257 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(213), op: ExtentClose { repair_id: ReconciliationId(213), extent_id: 68 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.257 INFO [0] client ExtentClose { repair_id: ReconciliationId(213), extent_id: 68 }
Sep 22 23:22:18.257 INFO [1] received reconcile message
Sep 22 23:22:18.257 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(213), op: ExtentClose { repair_id: ReconciliationId(213), extent_id: 68 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.257 INFO [1] client ExtentClose { repair_id: ReconciliationId(213), extent_id: 68 }
Sep 22 23:22:18.257 INFO [2] received reconcile message
Sep 22 23:22:18.257 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(213), op: ExtentClose { repair_id: ReconciliationId(213), extent_id: 68 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.257 INFO [2] client ExtentClose { repair_id: ReconciliationId(213), extent_id: 68 }
Sep 22 23:22:18.257 DEBG 213 Close extent 68
Sep 22 23:22:18.258 DEBG 213 Close extent 68
Sep 22 23:22:18.258 DEBG 213 Close extent 68
Sep 22 23:22:18.258 DEBG [2] It's time to notify for 213
Sep 22 23:22:18.258 INFO Completion from [2] id:213 status:true
Sep 22 23:22:18.258 INFO [214/752] Repair commands completed
Sep 22 23:22:18.258 INFO Pop front: ReconcileIO { id: ReconciliationId(214), op: ExtentRepair { repair_id: ReconciliationId(214), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.258 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.258 INFO [0] received reconcile message
Sep 22 23:22:18.258 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(214), op: ExtentRepair { repair_id: ReconciliationId(214), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.258 INFO [0] client ExtentRepair { repair_id: ReconciliationId(214), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.258 INFO [0] Sending repair request ReconciliationId(214)
Sep 22 23:22:18.258 INFO [1] received reconcile message
Sep 22 23:22:18.258 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(214), op: ExtentRepair { repair_id: ReconciliationId(214), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.259 INFO [1] client ExtentRepair { repair_id: ReconciliationId(214), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.259 INFO [1] No action required ReconciliationId(214)
Sep 22 23:22:18.259 INFO [2] received reconcile message
Sep 22 23:22:18.259 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(214), op: ExtentRepair { repair_id: ReconciliationId(214), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.259 INFO [2] client ExtentRepair { repair_id: ReconciliationId(214), extent_id: 68, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.259 INFO [2] No action required ReconciliationId(214)
Sep 22 23:22:18.259 DEBG 214 Repair extent 68 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.259 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/044.copy"
Sep 22 23:22:18.320 INFO accepted connection, remote_addr: 127.0.0.1:46505, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.320 TRCE incoming request, uri: /extent/68/files, method: GET, req_id: 0f06646f-470e-4234-97b4-bd55d0b4888e, remote_addr: 127.0.0.1:46505, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.320 INFO request completed, latency_us: 192, response_code: 200, uri: /extent/68/files, method: GET, req_id: 0f06646f-470e-4234-97b4-bd55d0b4888e, remote_addr: 127.0.0.1:46505, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.321 INFO eid:68 Found repair files: ["044", "044.db"]
Sep 22 23:22:18.321 TRCE incoming request, uri: /newextent/68/data, method: GET, req_id: 2ee0b688-b753-4cdb-9e08-1474ea08ff1e, remote_addr: 127.0.0.1:46505, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.321 INFO request completed, latency_us: 248, response_code: 200, uri: /newextent/68/data, method: GET, req_id: 2ee0b688-b753-4cdb-9e08-1474ea08ff1e, remote_addr: 127.0.0.1:46505, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.326 TRCE incoming request, uri: /newextent/68/db, method: GET, req_id: 086cd728-4b80-4448-b637-5d00fbc108df, remote_addr: 127.0.0.1:46505, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.326 INFO request completed, latency_us: 289, response_code: 200, uri: /newextent/68/db, method: GET, req_id: 086cd728-4b80-4448-b637-5d00fbc108df, remote_addr: 127.0.0.1:46505, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.327 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/044.copy" to "/tmp/downstairs-zrMnlo6G/00/000/044.replace"
Sep 22 23:22:18.327 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.328 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/044.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.328 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/044"
Sep 22 23:22:18.329 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/044.db"
Sep 22 23:22:18.329 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.329 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/044.replace" to "/tmp/downstairs-zrMnlo6G/00/000/044.completed"
Sep 22 23:22:18.329 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.329 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.329 DEBG [0] It's time to notify for 214
Sep 22 23:22:18.329 INFO Completion from [0] id:214 status:true
Sep 22 23:22:18.329 INFO [215/752] Repair commands completed
Sep 22 23:22:18.329 INFO Pop front: ReconcileIO { id: ReconciliationId(215), op: ExtentReopen { repair_id: ReconciliationId(215), extent_id: 68 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.329 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.329 INFO [0] received reconcile message
Sep 22 23:22:18.329 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(215), op: ExtentReopen { repair_id: ReconciliationId(215), extent_id: 68 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.329 INFO [0] client ExtentReopen { repair_id: ReconciliationId(215), extent_id: 68 }
Sep 22 23:22:18.329 INFO [1] received reconcile message
Sep 22 23:22:18.329 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(215), op: ExtentReopen { repair_id: ReconciliationId(215), extent_id: 68 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.329 INFO [1] client ExtentReopen { repair_id: ReconciliationId(215), extent_id: 68 }
Sep 22 23:22:18.329 INFO [2] received reconcile message
Sep 22 23:22:18.329 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(215), op: ExtentReopen { repair_id: ReconciliationId(215), extent_id: 68 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.329 INFO [2] client ExtentReopen { repair_id: ReconciliationId(215), extent_id: 68 }
Sep 22 23:22:18.329 DEBG 215 Reopen extent 68
Sep 22 23:22:18.330 DEBG 215 Reopen extent 68
Sep 22 23:22:18.330 DEBG 215 Reopen extent 68
Sep 22 23:22:18.331 DEBG [2] It's time to notify for 215
Sep 22 23:22:18.331 INFO Completion from [2] id:215 status:true
Sep 22 23:22:18.331 INFO [216/752] Repair commands completed
Sep 22 23:22:18.331 INFO Pop front: ReconcileIO { id: ReconciliationId(216), op: ExtentFlush { repair_id: ReconciliationId(216), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.331 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.331 INFO [0] received reconcile message
Sep 22 23:22:18.331 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(216), op: ExtentFlush { repair_id: ReconciliationId(216), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.331 INFO [0] client ExtentFlush { repair_id: ReconciliationId(216), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.331 INFO [1] received reconcile message
Sep 22 23:22:18.331 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(216), op: ExtentFlush { repair_id: ReconciliationId(216), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.331 INFO [1] client ExtentFlush { repair_id: ReconciliationId(216), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.331 INFO [2] received reconcile message
Sep 22 23:22:18.331 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(216), op: ExtentFlush { repair_id: ReconciliationId(216), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.331 INFO [2] client ExtentFlush { repair_id: ReconciliationId(216), extent_id: 123, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.331 DEBG 216 Flush extent 123 with f:2 g:2
Sep 22 23:22:18.331 DEBG Flush just extent 123 with f:2 and g:2
Sep 22 23:22:18.332 DEBG [1] It's time to notify for 216
Sep 22 23:22:18.332 INFO Completion from [1] id:216 status:true
Sep 22 23:22:18.332 INFO [217/752] Repair commands completed
Sep 22 23:22:18.332 INFO Pop front: ReconcileIO { id: ReconciliationId(217), op: ExtentClose { repair_id: ReconciliationId(217), extent_id: 123 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.332 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.332 INFO [0] received reconcile message
Sep 22 23:22:18.332 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(217), op: ExtentClose { repair_id: ReconciliationId(217), extent_id: 123 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.332 INFO [0] client ExtentClose { repair_id: ReconciliationId(217), extent_id: 123 }
Sep 22 23:22:18.332 INFO [1] received reconcile message
Sep 22 23:22:18.332 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(217), op: ExtentClose { repair_id: ReconciliationId(217), extent_id: 123 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.332 INFO [1] client ExtentClose { repair_id: ReconciliationId(217), extent_id: 123 }
Sep 22 23:22:18.332 INFO [2] received reconcile message
Sep 22 23:22:18.332 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(217), op: ExtentClose { repair_id: ReconciliationId(217), extent_id: 123 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.332 INFO [2] client ExtentClose { repair_id: ReconciliationId(217), extent_id: 123 }
Sep 22 23:22:18.332 DEBG 217 Close extent 123
Sep 22 23:22:18.332 DEBG 217 Close extent 123
Sep 22 23:22:18.333 DEBG 217 Close extent 123
Sep 22 23:22:18.333 DEBG [2] It's time to notify for 217
Sep 22 23:22:18.333 INFO Completion from [2] id:217 status:true
Sep 22 23:22:18.333 INFO [218/752] Repair commands completed
Sep 22 23:22:18.333 INFO Pop front: ReconcileIO { id: ReconciliationId(218), op: ExtentRepair { repair_id: ReconciliationId(218), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.333 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.333 INFO [0] received reconcile message
Sep 22 23:22:18.333 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(218), op: ExtentRepair { repair_id: ReconciliationId(218), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.333 INFO [0] client ExtentRepair { repair_id: ReconciliationId(218), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.333 INFO [0] Sending repair request ReconciliationId(218)
Sep 22 23:22:18.333 INFO [1] received reconcile message
Sep 22 23:22:18.333 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(218), op: ExtentRepair { repair_id: ReconciliationId(218), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.333 INFO [1] client ExtentRepair { repair_id: ReconciliationId(218), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.333 INFO [1] No action required ReconciliationId(218)
Sep 22 23:22:18.333 INFO [2] received reconcile message
Sep 22 23:22:18.333 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(218), op: ExtentRepair { repair_id: ReconciliationId(218), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.333 INFO [2] client ExtentRepair { repair_id: ReconciliationId(218), extent_id: 123, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.333 INFO [2] No action required ReconciliationId(218)
Sep 22 23:22:18.333 DEBG 218 Repair extent 123 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.333 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/07B.copy"
Sep 22 23:22:18.398 INFO accepted connection, remote_addr: 127.0.0.1:61337, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.398 TRCE incoming request, uri: /extent/123/files, method: GET, req_id: c8fee53d-ebe6-4f07-89f5-e0c768ed804f, remote_addr: 127.0.0.1:61337, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.399 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/123/files, method: GET, req_id: c8fee53d-ebe6-4f07-89f5-e0c768ed804f, remote_addr: 127.0.0.1:61337, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.399 INFO eid:123 Found repair files: ["07B", "07B.db"]
Sep 22 23:22:18.399 TRCE incoming request, uri: /newextent/123/data, method: GET, req_id: 95cf2971-edd6-4637-9a8f-01c6322fecab, remote_addr: 127.0.0.1:61337, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.400 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/123/data, method: GET, req_id: 95cf2971-edd6-4637-9a8f-01c6322fecab, remote_addr: 127.0.0.1:61337, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.405 TRCE incoming request, uri: /newextent/123/db, method: GET, req_id: fded857c-e2a8-421f-a8ba-32b10339259c, remote_addr: 127.0.0.1:61337, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.405 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/123/db, method: GET, req_id: fded857c-e2a8-421f-a8ba-32b10339259c, remote_addr: 127.0.0.1:61337, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.406 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/07B.copy" to "/tmp/downstairs-zrMnlo6G/00/000/07B.replace"
Sep 22 23:22:18.406 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.407 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/07B.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.407 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07B"
Sep 22 23:22:18.407 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07B.db"
Sep 22 23:22:18.407 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.407 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/07B.replace" to "/tmp/downstairs-zrMnlo6G/00/000/07B.completed"
Sep 22 23:22:18.407 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.407 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.407 DEBG [0] It's time to notify for 218
Sep 22 23:22:18.408 INFO Completion from [0] id:218 status:true
Sep 22 23:22:18.408 INFO [219/752] Repair commands completed
Sep 22 23:22:18.408 INFO Pop front: ReconcileIO { id: ReconciliationId(219), op: ExtentReopen { repair_id: ReconciliationId(219), extent_id: 123 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.408 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.408 INFO [0] received reconcile message
Sep 22 23:22:18.408 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(219), op: ExtentReopen { repair_id: ReconciliationId(219), extent_id: 123 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.408 INFO [0] client ExtentReopen { repair_id: ReconciliationId(219), extent_id: 123 }
Sep 22 23:22:18.408 INFO [1] received reconcile message
Sep 22 23:22:18.408 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(219), op: ExtentReopen { repair_id: ReconciliationId(219), extent_id: 123 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.408 INFO [1] client ExtentReopen { repair_id: ReconciliationId(219), extent_id: 123 }
Sep 22 23:22:18.408 INFO [2] received reconcile message
Sep 22 23:22:18.408 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(219), op: ExtentReopen { repair_id: ReconciliationId(219), extent_id: 123 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.408 INFO [2] client ExtentReopen { repair_id: ReconciliationId(219), extent_id: 123 }
Sep 22 23:22:18.408 DEBG 219 Reopen extent 123
Sep 22 23:22:18.409 DEBG 219 Reopen extent 123
Sep 22 23:22:18.409 DEBG 219 Reopen extent 123
Sep 22 23:22:18.410 DEBG [2] It's time to notify for 219
Sep 22 23:22:18.410 INFO Completion from [2] id:219 status:true
Sep 22 23:22:18.410 INFO [220/752] Repair commands completed
Sep 22 23:22:18.410 INFO Pop front: ReconcileIO { id: ReconciliationId(220), op: ExtentFlush { repair_id: ReconciliationId(220), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.410 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.410 INFO [0] received reconcile message
Sep 22 23:22:18.410 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(220), op: ExtentFlush { repair_id: ReconciliationId(220), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.410 INFO [0] client ExtentFlush { repair_id: ReconciliationId(220), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.410 INFO [1] received reconcile message
Sep 22 23:22:18.410 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(220), op: ExtentFlush { repair_id: ReconciliationId(220), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.410 INFO [1] client ExtentFlush { repair_id: ReconciliationId(220), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.410 INFO [2] received reconcile message
Sep 22 23:22:18.410 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(220), op: ExtentFlush { repair_id: ReconciliationId(220), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.410 INFO [2] client ExtentFlush { repair_id: ReconciliationId(220), extent_id: 55, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.410 DEBG 220 Flush extent 55 with f:2 g:2
Sep 22 23:22:18.410 DEBG Flush just extent 55 with f:2 and g:2
Sep 22 23:22:18.410 DEBG [1] It's time to notify for 220
Sep 22 23:22:18.410 INFO Completion from [1] id:220 status:true
Sep 22 23:22:18.410 INFO [221/752] Repair commands completed
Sep 22 23:22:18.410 INFO Pop front: ReconcileIO { id: ReconciliationId(221), op: ExtentClose { repair_id: ReconciliationId(221), extent_id: 55 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.410 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.410 INFO [0] received reconcile message
Sep 22 23:22:18.410 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(221), op: ExtentClose { repair_id: ReconciliationId(221), extent_id: 55 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.410 INFO [0] client ExtentClose { repair_id: ReconciliationId(221), extent_id: 55 }
Sep 22 23:22:18.410 INFO [1] received reconcile message
Sep 22 23:22:18.410 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(221), op: ExtentClose { repair_id: ReconciliationId(221), extent_id: 55 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.410 INFO [1] client ExtentClose { repair_id: ReconciliationId(221), extent_id: 55 }
Sep 22 23:22:18.411 INFO [2] received reconcile message
Sep 22 23:22:18.411 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(221), op: ExtentClose { repair_id: ReconciliationId(221), extent_id: 55 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.411 INFO [2] client ExtentClose { repair_id: ReconciliationId(221), extent_id: 55 }
Sep 22 23:22:18.411 DEBG 221 Close extent 55
Sep 22 23:22:18.411 DEBG 221 Close extent 55
Sep 22 23:22:18.411 DEBG 221 Close extent 55
Sep 22 23:22:18.412 DEBG [2] It's time to notify for 221
Sep 22 23:22:18.412 INFO Completion from [2] id:221 status:true
Sep 22 23:22:18.412 INFO [222/752] Repair commands completed
Sep 22 23:22:18.412 INFO Pop front: ReconcileIO { id: ReconciliationId(222), op: ExtentRepair { repair_id: ReconciliationId(222), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.412 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.412 INFO [0] received reconcile message
Sep 22 23:22:18.412 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(222), op: ExtentRepair { repair_id: ReconciliationId(222), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.412 INFO [0] client ExtentRepair { repair_id: ReconciliationId(222), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.412 INFO [0] Sending repair request ReconciliationId(222)
Sep 22 23:22:18.412 INFO [1] received reconcile message
Sep 22 23:22:18.412 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(222), op: ExtentRepair { repair_id: ReconciliationId(222), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.412 INFO [1] client ExtentRepair { repair_id: ReconciliationId(222), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.412 INFO [1] No action required ReconciliationId(222)
Sep 22 23:22:18.412 INFO [2] received reconcile message
Sep 22 23:22:18.412 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(222), op: ExtentRepair { repair_id: ReconciliationId(222), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.412 INFO [2] client ExtentRepair { repair_id: ReconciliationId(222), extent_id: 55, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.412 INFO [2] No action required ReconciliationId(222)
Sep 22 23:22:18.412 DEBG 222 Repair extent 55 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.412 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/037.copy"
Sep 22 23:22:18.475 INFO accepted connection, remote_addr: 127.0.0.1:52619, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.476 TRCE incoming request, uri: /extent/55/files, method: GET, req_id: b9421453-5d39-43d3-ba0b-08e7d720653d, remote_addr: 127.0.0.1:52619, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.476 INFO request completed, latency_us: 189, response_code: 200, uri: /extent/55/files, method: GET, req_id: b9421453-5d39-43d3-ba0b-08e7d720653d, remote_addr: 127.0.0.1:52619, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.476 INFO eid:55 Found repair files: ["037", "037.db"]
Sep 22 23:22:18.476 TRCE incoming request, uri: /newextent/55/data, method: GET, req_id: d0513051-e217-4cf7-bdb9-05812f4bf8e8, remote_addr: 127.0.0.1:52619, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.477 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/55/data, method: GET, req_id: d0513051-e217-4cf7-bdb9-05812f4bf8e8, remote_addr: 127.0.0.1:52619, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.482 TRCE incoming request, uri: /newextent/55/db, method: GET, req_id: b0f27a1b-a0ae-4437-b37c-3ad59ccb1ac1, remote_addr: 127.0.0.1:52619, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.482 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/55/db, method: GET, req_id: b0f27a1b-a0ae-4437-b37c-3ad59ccb1ac1, remote_addr: 127.0.0.1:52619, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.483 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/037.copy" to "/tmp/downstairs-zrMnlo6G/00/000/037.replace"
Sep 22 23:22:18.483 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.484 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/037.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.484 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/037"
Sep 22 23:22:18.484 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/037.db"
Sep 22 23:22:18.484 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.484 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/037.replace" to "/tmp/downstairs-zrMnlo6G/00/000/037.completed"
Sep 22 23:22:18.484 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.484 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.485 DEBG [0] It's time to notify for 222
Sep 22 23:22:18.485 INFO Completion from [0] id:222 status:true
Sep 22 23:22:18.485 INFO [223/752] Repair commands completed
Sep 22 23:22:18.485 INFO Pop front: ReconcileIO { id: ReconciliationId(223), op: ExtentReopen { repair_id: ReconciliationId(223), extent_id: 55 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.485 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.485 INFO [0] received reconcile message
Sep 22 23:22:18.485 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(223), op: ExtentReopen { repair_id: ReconciliationId(223), extent_id: 55 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.485 INFO [0] client ExtentReopen { repair_id: ReconciliationId(223), extent_id: 55 }
Sep 22 23:22:18.485 INFO [1] received reconcile message
Sep 22 23:22:18.485 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(223), op: ExtentReopen { repair_id: ReconciliationId(223), extent_id: 55 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.485 INFO [1] client ExtentReopen { repair_id: ReconciliationId(223), extent_id: 55 }
Sep 22 23:22:18.485 INFO [2] received reconcile message
Sep 22 23:22:18.485 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(223), op: ExtentReopen { repair_id: ReconciliationId(223), extent_id: 55 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.485 INFO [2] client ExtentReopen { repair_id: ReconciliationId(223), extent_id: 55 }
Sep 22 23:22:18.485 DEBG 223 Reopen extent 55
Sep 22 23:22:18.486 DEBG 223 Reopen extent 55
Sep 22 23:22:18.486 DEBG 223 Reopen extent 55
Sep 22 23:22:18.487 DEBG [2] It's time to notify for 223
Sep 22 23:22:18.487 INFO Completion from [2] id:223 status:true
Sep 22 23:22:18.487 INFO [224/752] Repair commands completed
Sep 22 23:22:18.487 INFO Pop front: ReconcileIO { id: ReconciliationId(224), op: ExtentFlush { repair_id: ReconciliationId(224), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.487 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.487 INFO [0] received reconcile message
Sep 22 23:22:18.487 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(224), op: ExtentFlush { repair_id: ReconciliationId(224), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.487 INFO [0] client ExtentFlush { repair_id: ReconciliationId(224), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.487 INFO [1] received reconcile message
Sep 22 23:22:18.487 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(224), op: ExtentFlush { repair_id: ReconciliationId(224), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.487 INFO [1] client ExtentFlush { repair_id: ReconciliationId(224), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.487 INFO [2] received reconcile message
Sep 22 23:22:18.487 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(224), op: ExtentFlush { repair_id: ReconciliationId(224), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.487 INFO [2] client ExtentFlush { repair_id: ReconciliationId(224), extent_id: 159, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.487 DEBG 224 Flush extent 159 with f:2 g:2
Sep 22 23:22:18.487 DEBG Flush just extent 159 with f:2 and g:2
Sep 22 23:22:18.487 DEBG [1] It's time to notify for 224
Sep 22 23:22:18.487 INFO Completion from [1] id:224 status:true
Sep 22 23:22:18.487 INFO [225/752] Repair commands completed
Sep 22 23:22:18.487 INFO Pop front: ReconcileIO { id: ReconciliationId(225), op: ExtentClose { repair_id: ReconciliationId(225), extent_id: 159 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.487 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.487 INFO [0] received reconcile message
Sep 22 23:22:18.487 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(225), op: ExtentClose { repair_id: ReconciliationId(225), extent_id: 159 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.487 INFO [0] client ExtentClose { repair_id: ReconciliationId(225), extent_id: 159 }
Sep 22 23:22:18.487 INFO [1] received reconcile message
Sep 22 23:22:18.487 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(225), op: ExtentClose { repair_id: ReconciliationId(225), extent_id: 159 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.487 INFO [1] client ExtentClose { repair_id: ReconciliationId(225), extent_id: 159 }
Sep 22 23:22:18.488 INFO [2] received reconcile message
Sep 22 23:22:18.488 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(225), op: ExtentClose { repair_id: ReconciliationId(225), extent_id: 159 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.488 INFO [2] client ExtentClose { repair_id: ReconciliationId(225), extent_id: 159 }
Sep 22 23:22:18.488 DEBG 225 Close extent 159
Sep 22 23:22:18.488 DEBG 225 Close extent 159
Sep 22 23:22:18.488 DEBG 225 Close extent 159
Sep 22 23:22:18.489 DEBG [2] It's time to notify for 225
Sep 22 23:22:18.489 INFO Completion from [2] id:225 status:true
Sep 22 23:22:18.489 INFO [226/752] Repair commands completed
Sep 22 23:22:18.489 INFO Pop front: ReconcileIO { id: ReconciliationId(226), op: ExtentRepair { repair_id: ReconciliationId(226), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.489 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.489 INFO [0] received reconcile message
Sep 22 23:22:18.489 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(226), op: ExtentRepair { repair_id: ReconciliationId(226), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.489 INFO [0] client ExtentRepair { repair_id: ReconciliationId(226), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.489 INFO [0] Sending repair request ReconciliationId(226)
Sep 22 23:22:18.489 INFO [1] received reconcile message
Sep 22 23:22:18.489 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(226), op: ExtentRepair { repair_id: ReconciliationId(226), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.489 INFO [1] client ExtentRepair { repair_id: ReconciliationId(226), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.489 INFO [1] No action required ReconciliationId(226)
Sep 22 23:22:18.489 INFO [2] received reconcile message
Sep 22 23:22:18.489 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(226), op: ExtentRepair { repair_id: ReconciliationId(226), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.489 INFO [2] client ExtentRepair { repair_id: ReconciliationId(226), extent_id: 159, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.489 INFO [2] No action required ReconciliationId(226)
Sep 22 23:22:18.489 DEBG 226 Repair extent 159 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.489 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/09F.copy"
Sep 22 23:22:18.552 INFO accepted connection, remote_addr: 127.0.0.1:64824, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.552 TRCE incoming request, uri: /extent/159/files, method: GET, req_id: a2c4a738-936c-489a-a497-bd2593b836bb, remote_addr: 127.0.0.1:64824, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.552 INFO request completed, latency_us: 188, response_code: 200, uri: /extent/159/files, method: GET, req_id: a2c4a738-936c-489a-a497-bd2593b836bb, remote_addr: 127.0.0.1:64824, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.553 INFO eid:159 Found repair files: ["09F", "09F.db"]
Sep 22 23:22:18.553 TRCE incoming request, uri: /newextent/159/data, method: GET, req_id: d3012b5b-4822-4e04-b273-924d66b8e4c5, remote_addr: 127.0.0.1:64824, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.553 INFO request completed, latency_us: 248, response_code: 200, uri: /newextent/159/data, method: GET, req_id: d3012b5b-4822-4e04-b273-924d66b8e4c5, remote_addr: 127.0.0.1:64824, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.558 TRCE incoming request, uri: /newextent/159/db, method: GET, req_id: 978c50d1-4d92-4168-a163-f0f57ce68012, remote_addr: 127.0.0.1:64824, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.559 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/159/db, method: GET, req_id: 978c50d1-4d92-4168-a163-f0f57ce68012, remote_addr: 127.0.0.1:64824, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.560 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/09F.copy" to "/tmp/downstairs-zrMnlo6G/00/000/09F.replace"
Sep 22 23:22:18.560 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.560 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/09F.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.561 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09F"
Sep 22 23:22:18.561 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09F.db"
Sep 22 23:22:18.561 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.561 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/09F.replace" to "/tmp/downstairs-zrMnlo6G/00/000/09F.completed"
Sep 22 23:22:18.561 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.561 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.561 DEBG [0] It's time to notify for 226
Sep 22 23:22:18.561 INFO Completion from [0] id:226 status:true
Sep 22 23:22:18.561 INFO [227/752] Repair commands completed
Sep 22 23:22:18.561 INFO Pop front: ReconcileIO { id: ReconciliationId(227), op: ExtentReopen { repair_id: ReconciliationId(227), extent_id: 159 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.561 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.561 INFO [0] received reconcile message
Sep 22 23:22:18.561 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(227), op: ExtentReopen { repair_id: ReconciliationId(227), extent_id: 159 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.561 INFO [0] client ExtentReopen { repair_id: ReconciliationId(227), extent_id: 159 }
Sep 22 23:22:18.561 INFO [1] received reconcile message
Sep 22 23:22:18.561 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(227), op: ExtentReopen { repair_id: ReconciliationId(227), extent_id: 159 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.561 INFO [1] client ExtentReopen { repair_id: ReconciliationId(227), extent_id: 159 }
Sep 22 23:22:18.561 INFO [2] received reconcile message
Sep 22 23:22:18.561 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(227), op: ExtentReopen { repair_id: ReconciliationId(227), extent_id: 159 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.562 INFO [2] client ExtentReopen { repair_id: ReconciliationId(227), extent_id: 159 }
Sep 22 23:22:18.562 DEBG 227 Reopen extent 159
Sep 22 23:22:18.562 DEBG 227 Reopen extent 159
Sep 22 23:22:18.563 DEBG 227 Reopen extent 159
Sep 22 23:22:18.563 DEBG [2] It's time to notify for 227
Sep 22 23:22:18.563 INFO Completion from [2] id:227 status:true
Sep 22 23:22:18.563 INFO [228/752] Repair commands completed
Sep 22 23:22:18.563 INFO Pop front: ReconcileIO { id: ReconciliationId(228), op: ExtentFlush { repair_id: ReconciliationId(228), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.563 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.563 INFO [0] received reconcile message
Sep 22 23:22:18.563 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(228), op: ExtentFlush { repair_id: ReconciliationId(228), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.563 INFO [0] client ExtentFlush { repair_id: ReconciliationId(228), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.563 INFO [1] received reconcile message
Sep 22 23:22:18.563 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(228), op: ExtentFlush { repair_id: ReconciliationId(228), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.563 INFO [1] client ExtentFlush { repair_id: ReconciliationId(228), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.563 INFO [2] received reconcile message
Sep 22 23:22:18.564 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(228), op: ExtentFlush { repair_id: ReconciliationId(228), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.564 INFO [2] client ExtentFlush { repair_id: ReconciliationId(228), extent_id: 35, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.564 DEBG 228 Flush extent 35 with f:2 g:2
Sep 22 23:22:18.564 DEBG Flush just extent 35 with f:2 and g:2
Sep 22 23:22:18.564 DEBG [1] It's time to notify for 228
Sep 22 23:22:18.564 INFO Completion from [1] id:228 status:true
Sep 22 23:22:18.564 INFO [229/752] Repair commands completed
Sep 22 23:22:18.564 INFO Pop front: ReconcileIO { id: ReconciliationId(229), op: ExtentClose { repair_id: ReconciliationId(229), extent_id: 35 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.564 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.564 INFO [0] received reconcile message
Sep 22 23:22:18.564 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(229), op: ExtentClose { repair_id: ReconciliationId(229), extent_id: 35 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.564 INFO [0] client ExtentClose { repair_id: ReconciliationId(229), extent_id: 35 }
Sep 22 23:22:18.564 INFO [1] received reconcile message
Sep 22 23:22:18.564 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(229), op: ExtentClose { repair_id: ReconciliationId(229), extent_id: 35 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.564 INFO [1] client ExtentClose { repair_id: ReconciliationId(229), extent_id: 35 }
Sep 22 23:22:18.564 INFO [2] received reconcile message
Sep 22 23:22:18.564 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(229), op: ExtentClose { repair_id: ReconciliationId(229), extent_id: 35 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.564 INFO [2] client ExtentClose { repair_id: ReconciliationId(229), extent_id: 35 }
Sep 22 23:22:18.564 DEBG 229 Close extent 35
Sep 22 23:22:18.564 DEBG 229 Close extent 35
Sep 22 23:22:18.565 DEBG 229 Close extent 35
Sep 22 23:22:18.565 DEBG [2] It's time to notify for 229
Sep 22 23:22:18.565 INFO Completion from [2] id:229 status:true
Sep 22 23:22:18.565 INFO [230/752] Repair commands completed
Sep 22 23:22:18.565 INFO Pop front: ReconcileIO { id: ReconciliationId(230), op: ExtentRepair { repair_id: ReconciliationId(230), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.565 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.565 INFO [0] received reconcile message
Sep 22 23:22:18.565 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(230), op: ExtentRepair { repair_id: ReconciliationId(230), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.565 INFO [0] client ExtentRepair { repair_id: ReconciliationId(230), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.565 INFO [0] Sending repair request ReconciliationId(230)
Sep 22 23:22:18.565 INFO [1] received reconcile message
Sep 22 23:22:18.565 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(230), op: ExtentRepair { repair_id: ReconciliationId(230), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.565 INFO [1] client ExtentRepair { repair_id: ReconciliationId(230), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.565 INFO [1] No action required ReconciliationId(230)
Sep 22 23:22:18.565 INFO [2] received reconcile message
Sep 22 23:22:18.565 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(230), op: ExtentRepair { repair_id: ReconciliationId(230), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.566 INFO [2] client ExtentRepair { repair_id: ReconciliationId(230), extent_id: 35, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.566 INFO [2] No action required ReconciliationId(230)
Sep 22 23:22:18.566 DEBG 230 Repair extent 35 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.566 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/023.copy"
Sep 22 23:22:18.629 INFO accepted connection, remote_addr: 127.0.0.1:40899, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.629 TRCE incoming request, uri: /extent/35/files, method: GET, req_id: 225c9d90-712b-43ef-b653-ea878429a7be, remote_addr: 127.0.0.1:40899, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.629 INFO request completed, latency_us: 230, response_code: 200, uri: /extent/35/files, method: GET, req_id: 225c9d90-712b-43ef-b653-ea878429a7be, remote_addr: 127.0.0.1:40899, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.629 INFO eid:35 Found repair files: ["023", "023.db"]
Sep 22 23:22:18.630 TRCE incoming request, uri: /newextent/35/data, method: GET, req_id: 3a9cd04a-dae9-4562-9e96-dce1548ec2fb, remote_addr: 127.0.0.1:40899, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.630 INFO request completed, latency_us: 335, response_code: 200, uri: /newextent/35/data, method: GET, req_id: 3a9cd04a-dae9-4562-9e96-dce1548ec2fb, remote_addr: 127.0.0.1:40899, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.635 TRCE incoming request, uri: /newextent/35/db, method: GET, req_id: 4291d16b-a54b-4265-9470-27718c042181, remote_addr: 127.0.0.1:40899, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.636 INFO request completed, latency_us: 294, response_code: 200, uri: /newextent/35/db, method: GET, req_id: 4291d16b-a54b-4265-9470-27718c042181, remote_addr: 127.0.0.1:40899, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.637 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/023.copy" to "/tmp/downstairs-zrMnlo6G/00/000/023.replace"
Sep 22 23:22:18.637 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.638 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/023.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.638 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/023"
Sep 22 23:22:18.639 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/023.db"
Sep 22 23:22:18.639 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.639 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/023.replace" to "/tmp/downstairs-zrMnlo6G/00/000/023.completed"
Sep 22 23:22:18.639 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.639 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.639 DEBG [0] It's time to notify for 230
Sep 22 23:22:18.639 INFO Completion from [0] id:230 status:true
Sep 22 23:22:18.639 INFO [231/752] Repair commands completed
Sep 22 23:22:18.639 INFO Pop front: ReconcileIO { id: ReconciliationId(231), op: ExtentReopen { repair_id: ReconciliationId(231), extent_id: 35 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.639 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.639 INFO [0] received reconcile message
Sep 22 23:22:18.639 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(231), op: ExtentReopen { repair_id: ReconciliationId(231), extent_id: 35 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.639 INFO [0] client ExtentReopen { repair_id: ReconciliationId(231), extent_id: 35 }
Sep 22 23:22:18.639 INFO [1] received reconcile message
Sep 22 23:22:18.639 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(231), op: ExtentReopen { repair_id: ReconciliationId(231), extent_id: 35 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.639 INFO [1] client ExtentReopen { repair_id: ReconciliationId(231), extent_id: 35 }
Sep 22 23:22:18.639 INFO [2] received reconcile message
Sep 22 23:22:18.639 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(231), op: ExtentReopen { repair_id: ReconciliationId(231), extent_id: 35 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.639 INFO [2] client ExtentReopen { repair_id: ReconciliationId(231), extent_id: 35 }
Sep 22 23:22:18.640 DEBG 231 Reopen extent 35
Sep 22 23:22:18.640 DEBG 231 Reopen extent 35
Sep 22 23:22:18.641 DEBG 231 Reopen extent 35
Sep 22 23:22:18.642 DEBG [2] It's time to notify for 231
Sep 22 23:22:18.642 INFO Completion from [2] id:231 status:true
Sep 22 23:22:18.642 INFO [232/752] Repair commands completed
Sep 22 23:22:18.642 INFO Pop front: ReconcileIO { id: ReconciliationId(232), op: ExtentFlush { repair_id: ReconciliationId(232), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.642 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.642 INFO [0] received reconcile message
Sep 22 23:22:18.642 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(232), op: ExtentFlush { repair_id: ReconciliationId(232), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.642 INFO [0] client ExtentFlush { repair_id: ReconciliationId(232), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.642 INFO [1] received reconcile message
Sep 22 23:22:18.642 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(232), op: ExtentFlush { repair_id: ReconciliationId(232), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.642 INFO [1] client ExtentFlush { repair_id: ReconciliationId(232), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.642 INFO [2] received reconcile message
Sep 22 23:22:18.642 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(232), op: ExtentFlush { repair_id: ReconciliationId(232), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.642 INFO [2] client ExtentFlush { repair_id: ReconciliationId(232), extent_id: 162, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.642 DEBG 232 Flush extent 162 with f:2 g:2
Sep 22 23:22:18.642 DEBG Flush just extent 162 with f:2 and g:2
Sep 22 23:22:18.642 DEBG [1] It's time to notify for 232
Sep 22 23:22:18.642 INFO Completion from [1] id:232 status:true
Sep 22 23:22:18.642 INFO [233/752] Repair commands completed
Sep 22 23:22:18.642 INFO Pop front: ReconcileIO { id: ReconciliationId(233), op: ExtentClose { repair_id: ReconciliationId(233), extent_id: 162 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.642 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.642 INFO [0] received reconcile message
Sep 22 23:22:18.642 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(233), op: ExtentClose { repair_id: ReconciliationId(233), extent_id: 162 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.642 INFO [0] client ExtentClose { repair_id: ReconciliationId(233), extent_id: 162 }
Sep 22 23:22:18.642 INFO [1] received reconcile message
Sep 22 23:22:18.642 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(233), op: ExtentClose { repair_id: ReconciliationId(233), extent_id: 162 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.642 INFO [1] client ExtentClose { repair_id: ReconciliationId(233), extent_id: 162 }
Sep 22 23:22:18.642 INFO [2] received reconcile message
Sep 22 23:22:18.642 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(233), op: ExtentClose { repair_id: ReconciliationId(233), extent_id: 162 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.642 INFO [2] client ExtentClose { repair_id: ReconciliationId(233), extent_id: 162 }
Sep 22 23:22:18.643 DEBG 233 Close extent 162
Sep 22 23:22:18.643 DEBG 233 Close extent 162
Sep 22 23:22:18.643 DEBG 233 Close extent 162
Sep 22 23:22:18.644 DEBG [2] It's time to notify for 233
Sep 22 23:22:18.644 INFO Completion from [2] id:233 status:true
Sep 22 23:22:18.644 INFO [234/752] Repair commands completed
Sep 22 23:22:18.644 INFO Pop front: ReconcileIO { id: ReconciliationId(234), op: ExtentRepair { repair_id: ReconciliationId(234), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.644 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.644 INFO [0] received reconcile message
Sep 22 23:22:18.644 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(234), op: ExtentRepair { repair_id: ReconciliationId(234), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.644 INFO [0] client ExtentRepair { repair_id: ReconciliationId(234), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.644 INFO [0] Sending repair request ReconciliationId(234)
Sep 22 23:22:18.644 INFO [1] received reconcile message
Sep 22 23:22:18.644 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(234), op: ExtentRepair { repair_id: ReconciliationId(234), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.644 INFO [1] client ExtentRepair { repair_id: ReconciliationId(234), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.644 INFO [1] No action required ReconciliationId(234)
Sep 22 23:22:18.644 INFO [2] received reconcile message
Sep 22 23:22:18.644 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(234), op: ExtentRepair { repair_id: ReconciliationId(234), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.644 INFO [2] client ExtentRepair { repair_id: ReconciliationId(234), extent_id: 162, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.644 INFO [2] No action required ReconciliationId(234)
Sep 22 23:22:18.644 DEBG 234 Repair extent 162 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.644 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0A2.copy"
Sep 22 23:22:18.648 ERRO [0] job id 1078 saw error GenericError("test error")
Sep 22 23:22:18.648 DEBG [rc] retire 1078 clears [JobId(1077), JobId(1078)], : downstairs
Sep 22 23:22:18.648 DEBG IO Flush 1080 has deps [JobId(1079)]
Sep 22 23:22:18.654 DEBG Read      :1079 deps:[JobId(1078)] res:true
Sep 22 23:22:18.678 INFO [lossy] sleeping 1 second
Sep 22 23:22:18.709 INFO accepted connection, remote_addr: 127.0.0.1:44201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.710 TRCE incoming request, uri: /extent/162/files, method: GET, req_id: e9c28114-5c07-4f57-9b84-76579bb9f49c, remote_addr: 127.0.0.1:44201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.710 INFO request completed, latency_us: 275, response_code: 200, uri: /extent/162/files, method: GET, req_id: e9c28114-5c07-4f57-9b84-76579bb9f49c, remote_addr: 127.0.0.1:44201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.710 INFO eid:162 Found repair files: ["0A2", "0A2.db"]
Sep 22 23:22:18.711 TRCE incoming request, uri: /newextent/162/data, method: GET, req_id: 16da6fda-565a-46d7-a395-65524eafb833, remote_addr: 127.0.0.1:44201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.711 INFO request completed, latency_us: 331, response_code: 200, uri: /newextent/162/data, method: GET, req_id: 16da6fda-565a-46d7-a395-65524eafb833, remote_addr: 127.0.0.1:44201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.716 TRCE incoming request, uri: /newextent/162/db, method: GET, req_id: 8799dd52-b6f0-4e20-8695-d873cff8edca, remote_addr: 127.0.0.1:44201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.717 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/162/db, method: GET, req_id: 8799dd52-b6f0-4e20-8695-d873cff8edca, remote_addr: 127.0.0.1:44201, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.718 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0A2.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0A2.replace"
Sep 22 23:22:18.718 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.719 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0A2.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.719 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A2"
Sep 22 23:22:18.719 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A2.db"
Sep 22 23:22:18.719 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.719 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0A2.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0A2.completed"
Sep 22 23:22:18.719 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.720 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.720 DEBG [0] It's time to notify for 234
Sep 22 23:22:18.720 INFO Completion from [0] id:234 status:true
Sep 22 23:22:18.720 INFO [235/752] Repair commands completed
Sep 22 23:22:18.720 INFO Pop front: ReconcileIO { id: ReconciliationId(235), op: ExtentReopen { repair_id: ReconciliationId(235), extent_id: 162 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.720 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.720 INFO [0] received reconcile message
Sep 22 23:22:18.720 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(235), op: ExtentReopen { repair_id: ReconciliationId(235), extent_id: 162 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.720 INFO [0] client ExtentReopen { repair_id: ReconciliationId(235), extent_id: 162 }
Sep 22 23:22:18.720 INFO [1] received reconcile message
Sep 22 23:22:18.720 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(235), op: ExtentReopen { repair_id: ReconciliationId(235), extent_id: 162 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.720 INFO [1] client ExtentReopen { repair_id: ReconciliationId(235), extent_id: 162 }
Sep 22 23:22:18.720 INFO [2] received reconcile message
Sep 22 23:22:18.720 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(235), op: ExtentReopen { repair_id: ReconciliationId(235), extent_id: 162 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.720 INFO [2] client ExtentReopen { repair_id: ReconciliationId(235), extent_id: 162 }
Sep 22 23:22:18.721 DEBG 235 Reopen extent 162
Sep 22 23:22:18.722 DEBG 235 Reopen extent 162
Sep 22 23:22:18.722 DEBG 235 Reopen extent 162
Sep 22 23:22:18.723 DEBG [2] It's time to notify for 235
Sep 22 23:22:18.723 INFO Completion from [2] id:235 status:true
Sep 22 23:22:18.723 INFO [236/752] Repair commands completed
Sep 22 23:22:18.723 INFO Pop front: ReconcileIO { id: ReconciliationId(236), op: ExtentFlush { repair_id: ReconciliationId(236), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.723 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.723 INFO [0] received reconcile message
Sep 22 23:22:18.723 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(236), op: ExtentFlush { repair_id: ReconciliationId(236), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.723 INFO [0] client ExtentFlush { repair_id: ReconciliationId(236), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.724 INFO [1] received reconcile message
Sep 22 23:22:18.724 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(236), op: ExtentFlush { repair_id: ReconciliationId(236), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.724 INFO [1] client ExtentFlush { repair_id: ReconciliationId(236), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.724 INFO [2] received reconcile message
Sep 22 23:22:18.724 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(236), op: ExtentFlush { repair_id: ReconciliationId(236), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.724 INFO [2] client ExtentFlush { repair_id: ReconciliationId(236), extent_id: 121, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.724 DEBG 236 Flush extent 121 with f:2 g:2
Sep 22 23:22:18.724 DEBG Flush just extent 121 with f:2 and g:2
Sep 22 23:22:18.724 DEBG [1] It's time to notify for 236
Sep 22 23:22:18.724 INFO Completion from [1] id:236 status:true
Sep 22 23:22:18.724 INFO [237/752] Repair commands completed
Sep 22 23:22:18.724 INFO Pop front: ReconcileIO { id: ReconciliationId(237), op: ExtentClose { repair_id: ReconciliationId(237), extent_id: 121 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.724 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.724 INFO [0] received reconcile message
Sep 22 23:22:18.724 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(237), op: ExtentClose { repair_id: ReconciliationId(237), extent_id: 121 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.724 INFO [0] client ExtentClose { repair_id: ReconciliationId(237), extent_id: 121 }
Sep 22 23:22:18.724 INFO [1] received reconcile message
Sep 22 23:22:18.724 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(237), op: ExtentClose { repair_id: ReconciliationId(237), extent_id: 121 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.724 INFO [1] client ExtentClose { repair_id: ReconciliationId(237), extent_id: 121 }
Sep 22 23:22:18.724 INFO [2] received reconcile message
Sep 22 23:22:18.724 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(237), op: ExtentClose { repair_id: ReconciliationId(237), extent_id: 121 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.724 INFO [2] client ExtentClose { repair_id: ReconciliationId(237), extent_id: 121 }
Sep 22 23:22:18.725 DEBG 237 Close extent 121
Sep 22 23:22:18.725 DEBG 237 Close extent 121
Sep 22 23:22:18.725 DEBG 237 Close extent 121
Sep 22 23:22:18.726 DEBG [2] It's time to notify for 237
Sep 22 23:22:18.726 INFO Completion from [2] id:237 status:true
Sep 22 23:22:18.726 INFO [238/752] Repair commands completed
Sep 22 23:22:18.726 INFO Pop front: ReconcileIO { id: ReconciliationId(238), op: ExtentRepair { repair_id: ReconciliationId(238), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.726 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.726 INFO [0] received reconcile message
Sep 22 23:22:18.726 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(238), op: ExtentRepair { repair_id: ReconciliationId(238), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.726 INFO [0] client ExtentRepair { repair_id: ReconciliationId(238), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.726 INFO [0] Sending repair request ReconciliationId(238)
Sep 22 23:22:18.726 INFO [1] received reconcile message
Sep 22 23:22:18.726 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(238), op: ExtentRepair { repair_id: ReconciliationId(238), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.726 INFO [1] client ExtentRepair { repair_id: ReconciliationId(238), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.726 INFO [1] No action required ReconciliationId(238)
Sep 22 23:22:18.726 INFO [2] received reconcile message
Sep 22 23:22:18.726 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(238), op: ExtentRepair { repair_id: ReconciliationId(238), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.726 INFO [2] client ExtentRepair { repair_id: ReconciliationId(238), extent_id: 121, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.727 INFO [2] No action required ReconciliationId(238)
Sep 22 23:22:18.727 DEBG 238 Repair extent 121 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.727 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/079.copy"
Sep 22 23:22:18.794 INFO accepted connection, remote_addr: 127.0.0.1:39641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.794 TRCE incoming request, uri: /extent/121/files, method: GET, req_id: 3970fe43-b992-4681-bcaf-6f28a33f63e8, remote_addr: 127.0.0.1:39641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.794 INFO request completed, latency_us: 251, response_code: 200, uri: /extent/121/files, method: GET, req_id: 3970fe43-b992-4681-bcaf-6f28a33f63e8, remote_addr: 127.0.0.1:39641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.795 INFO eid:121 Found repair files: ["079", "079.db"]
Sep 22 23:22:18.795 TRCE incoming request, uri: /newextent/121/data, method: GET, req_id: d390b511-5bff-4098-a403-ab8651c4b24e, remote_addr: 127.0.0.1:39641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.795 INFO request completed, latency_us: 340, response_code: 200, uri: /newextent/121/data, method: GET, req_id: d390b511-5bff-4098-a403-ab8651c4b24e, remote_addr: 127.0.0.1:39641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.800 TRCE incoming request, uri: /newextent/121/db, method: GET, req_id: 0fc18e80-1916-4f36-84d2-f7f5d8a9b47b, remote_addr: 127.0.0.1:39641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.801 INFO request completed, latency_us: 288, response_code: 200, uri: /newextent/121/db, method: GET, req_id: 0fc18e80-1916-4f36-84d2-f7f5d8a9b47b, remote_addr: 127.0.0.1:39641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.802 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/079.copy" to "/tmp/downstairs-zrMnlo6G/00/000/079.replace"
Sep 22 23:22:18.802 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.803 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/079.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.803 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/079"
Sep 22 23:22:18.803 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/079.db"
Sep 22 23:22:18.803 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.803 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/079.replace" to "/tmp/downstairs-zrMnlo6G/00/000/079.completed"
Sep 22 23:22:18.803 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.803 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.804 DEBG [0] It's time to notify for 238
Sep 22 23:22:18.804 INFO Completion from [0] id:238 status:true
Sep 22 23:22:18.804 INFO [239/752] Repair commands completed
Sep 22 23:22:18.804 INFO Pop front: ReconcileIO { id: ReconciliationId(239), op: ExtentReopen { repair_id: ReconciliationId(239), extent_id: 121 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.804 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.804 INFO [0] received reconcile message
Sep 22 23:22:18.804 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(239), op: ExtentReopen { repair_id: ReconciliationId(239), extent_id: 121 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.804 INFO [0] client ExtentReopen { repair_id: ReconciliationId(239), extent_id: 121 }
Sep 22 23:22:18.804 INFO [1] received reconcile message
Sep 22 23:22:18.804 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(239), op: ExtentReopen { repair_id: ReconciliationId(239), extent_id: 121 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.804 INFO [1] client ExtentReopen { repair_id: ReconciliationId(239), extent_id: 121 }
Sep 22 23:22:18.804 INFO [2] received reconcile message
Sep 22 23:22:18.804 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(239), op: ExtentReopen { repair_id: ReconciliationId(239), extent_id: 121 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.804 INFO [2] client ExtentReopen { repair_id: ReconciliationId(239), extent_id: 121 }
Sep 22 23:22:18.804 DEBG 239 Reopen extent 121
Sep 22 23:22:18.805 DEBG 239 Reopen extent 121
Sep 22 23:22:18.806 DEBG 239 Reopen extent 121
Sep 22 23:22:18.806 DEBG [2] It's time to notify for 239
Sep 22 23:22:18.806 INFO Completion from [2] id:239 status:true
Sep 22 23:22:18.806 INFO [240/752] Repair commands completed
Sep 22 23:22:18.806 INFO Pop front: ReconcileIO { id: ReconciliationId(240), op: ExtentFlush { repair_id: ReconciliationId(240), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.806 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.806 INFO [0] received reconcile message
Sep 22 23:22:18.806 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(240), op: ExtentFlush { repair_id: ReconciliationId(240), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.806 INFO [0] client ExtentFlush { repair_id: ReconciliationId(240), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.806 INFO [1] received reconcile message
Sep 22 23:22:18.806 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(240), op: ExtentFlush { repair_id: ReconciliationId(240), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.806 INFO [1] client ExtentFlush { repair_id: ReconciliationId(240), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.806 INFO [2] received reconcile message
Sep 22 23:22:18.806 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(240), op: ExtentFlush { repair_id: ReconciliationId(240), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.806 INFO [2] client ExtentFlush { repair_id: ReconciliationId(240), extent_id: 65, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.807 DEBG 240 Flush extent 65 with f:2 g:2
Sep 22 23:22:18.807 DEBG Flush just extent 65 with f:2 and g:2
Sep 22 23:22:18.807 DEBG [1] It's time to notify for 240
Sep 22 23:22:18.807 INFO Completion from [1] id:240 status:true
Sep 22 23:22:18.807 INFO [241/752] Repair commands completed
Sep 22 23:22:18.807 INFO Pop front: ReconcileIO { id: ReconciliationId(241), op: ExtentClose { repair_id: ReconciliationId(241), extent_id: 65 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.807 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.807 INFO [0] received reconcile message
Sep 22 23:22:18.807 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(241), op: ExtentClose { repair_id: ReconciliationId(241), extent_id: 65 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.807 INFO [0] client ExtentClose { repair_id: ReconciliationId(241), extent_id: 65 }
Sep 22 23:22:18.807 INFO [1] received reconcile message
Sep 22 23:22:18.807 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(241), op: ExtentClose { repair_id: ReconciliationId(241), extent_id: 65 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.807 INFO [1] client ExtentClose { repair_id: ReconciliationId(241), extent_id: 65 }
Sep 22 23:22:18.807 INFO [2] received reconcile message
Sep 22 23:22:18.807 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(241), op: ExtentClose { repair_id: ReconciliationId(241), extent_id: 65 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.807 INFO [2] client ExtentClose { repair_id: ReconciliationId(241), extent_id: 65 }
Sep 22 23:22:18.807 DEBG 241 Close extent 65
Sep 22 23:22:18.807 DEBG 241 Close extent 65
Sep 22 23:22:18.808 DEBG 241 Close extent 65
Sep 22 23:22:18.808 DEBG [2] It's time to notify for 241
Sep 22 23:22:18.808 INFO Completion from [2] id:241 status:true
Sep 22 23:22:18.808 INFO [242/752] Repair commands completed
Sep 22 23:22:18.808 INFO Pop front: ReconcileIO { id: ReconciliationId(242), op: ExtentRepair { repair_id: ReconciliationId(242), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.808 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.808 INFO [0] received reconcile message
Sep 22 23:22:18.808 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(242), op: ExtentRepair { repair_id: ReconciliationId(242), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.808 INFO [0] client ExtentRepair { repair_id: ReconciliationId(242), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.808 INFO [0] Sending repair request ReconciliationId(242)
Sep 22 23:22:18.808 INFO [1] received reconcile message
Sep 22 23:22:18.808 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(242), op: ExtentRepair { repair_id: ReconciliationId(242), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.808 INFO [1] client ExtentRepair { repair_id: ReconciliationId(242), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.808 INFO [1] No action required ReconciliationId(242)
Sep 22 23:22:18.808 INFO [2] received reconcile message
Sep 22 23:22:18.809 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(242), op: ExtentRepair { repair_id: ReconciliationId(242), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.809 INFO [2] client ExtentRepair { repair_id: ReconciliationId(242), extent_id: 65, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.809 INFO [2] No action required ReconciliationId(242)
Sep 22 23:22:18.809 DEBG 242 Repair extent 65 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.809 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/041.copy"
Sep 22 23:22:18.872 INFO accepted connection, remote_addr: 127.0.0.1:44802, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.873 TRCE incoming request, uri: /extent/65/files, method: GET, req_id: 8c7b47ce-948d-432f-9fc9-956864960b00, remote_addr: 127.0.0.1:44802, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.873 INFO request completed, latency_us: 272, response_code: 200, uri: /extent/65/files, method: GET, req_id: 8c7b47ce-948d-432f-9fc9-956864960b00, remote_addr: 127.0.0.1:44802, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.873 INFO eid:65 Found repair files: ["041", "041.db"]
Sep 22 23:22:18.874 TRCE incoming request, uri: /newextent/65/data, method: GET, req_id: 634dc24a-b102-4c66-a83f-7aea60b9d054, remote_addr: 127.0.0.1:44802, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.874 INFO request completed, latency_us: 363, response_code: 200, uri: /newextent/65/data, method: GET, req_id: 634dc24a-b102-4c66-a83f-7aea60b9d054, remote_addr: 127.0.0.1:44802, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.879 TRCE incoming request, uri: /newextent/65/db, method: GET, req_id: 7bd11843-9a6f-48bf-9f63-d26eff5d7842, remote_addr: 127.0.0.1:44802, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.879 INFO request completed, latency_us: 288, response_code: 200, uri: /newextent/65/db, method: GET, req_id: 7bd11843-9a6f-48bf-9f63-d26eff5d7842, remote_addr: 127.0.0.1:44802, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.880 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/041.copy" to "/tmp/downstairs-zrMnlo6G/00/000/041.replace"
Sep 22 23:22:18.880 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.882 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/041.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.882 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/041"
Sep 22 23:22:18.882 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/041.db"
Sep 22 23:22:18.882 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.882 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/041.replace" to "/tmp/downstairs-zrMnlo6G/00/000/041.completed"
Sep 22 23:22:18.882 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.882 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.882 DEBG [0] It's time to notify for 242
Sep 22 23:22:18.883 INFO Completion from [0] id:242 status:true
Sep 22 23:22:18.883 INFO [243/752] Repair commands completed
Sep 22 23:22:18.883 INFO Pop front: ReconcileIO { id: ReconciliationId(243), op: ExtentReopen { repair_id: ReconciliationId(243), extent_id: 65 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.883 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.883 INFO [0] received reconcile message
Sep 22 23:22:18.883 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(243), op: ExtentReopen { repair_id: ReconciliationId(243), extent_id: 65 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.883 INFO [0] client ExtentReopen { repair_id: ReconciliationId(243), extent_id: 65 }
Sep 22 23:22:18.883 INFO [1] received reconcile message
Sep 22 23:22:18.883 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(243), op: ExtentReopen { repair_id: ReconciliationId(243), extent_id: 65 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.883 INFO [1] client ExtentReopen { repair_id: ReconciliationId(243), extent_id: 65 }
Sep 22 23:22:18.883 INFO [2] received reconcile message
Sep 22 23:22:18.883 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(243), op: ExtentReopen { repair_id: ReconciliationId(243), extent_id: 65 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.883 INFO [2] client ExtentReopen { repair_id: ReconciliationId(243), extent_id: 65 }
Sep 22 23:22:18.883 DEBG 243 Reopen extent 65
Sep 22 23:22:18.884 DEBG 243 Reopen extent 65
Sep 22 23:22:18.884 DEBG 243 Reopen extent 65
Sep 22 23:22:18.885 DEBG [2] It's time to notify for 243
Sep 22 23:22:18.885 INFO Completion from [2] id:243 status:true
Sep 22 23:22:18.885 INFO [244/752] Repair commands completed
Sep 22 23:22:18.885 INFO Pop front: ReconcileIO { id: ReconciliationId(244), op: ExtentFlush { repair_id: ReconciliationId(244), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.885 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.885 INFO [0] received reconcile message
Sep 22 23:22:18.885 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(244), op: ExtentFlush { repair_id: ReconciliationId(244), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.885 INFO [0] client ExtentFlush { repair_id: ReconciliationId(244), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.885 INFO [1] received reconcile message
Sep 22 23:22:18.885 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(244), op: ExtentFlush { repair_id: ReconciliationId(244), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.885 INFO [1] client ExtentFlush { repair_id: ReconciliationId(244), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.885 INFO [2] received reconcile message
Sep 22 23:22:18.885 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(244), op: ExtentFlush { repair_id: ReconciliationId(244), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.885 INFO [2] client ExtentFlush { repair_id: ReconciliationId(244), extent_id: 106, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.885 DEBG 244 Flush extent 106 with f:2 g:2
Sep 22 23:22:18.885 DEBG Flush just extent 106 with f:2 and g:2
Sep 22 23:22:18.886 DEBG [1] It's time to notify for 244
Sep 22 23:22:18.886 INFO Completion from [1] id:244 status:true
Sep 22 23:22:18.886 INFO [245/752] Repair commands completed
Sep 22 23:22:18.886 INFO Pop front: ReconcileIO { id: ReconciliationId(245), op: ExtentClose { repair_id: ReconciliationId(245), extent_id: 106 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.886 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.886 INFO [0] received reconcile message
Sep 22 23:22:18.886 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(245), op: ExtentClose { repair_id: ReconciliationId(245), extent_id: 106 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.886 INFO [0] client ExtentClose { repair_id: ReconciliationId(245), extent_id: 106 }
Sep 22 23:22:18.886 INFO [1] received reconcile message
Sep 22 23:22:18.886 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(245), op: ExtentClose { repair_id: ReconciliationId(245), extent_id: 106 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.886 INFO [1] client ExtentClose { repair_id: ReconciliationId(245), extent_id: 106 }
Sep 22 23:22:18.886 INFO [2] received reconcile message
Sep 22 23:22:18.886 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(245), op: ExtentClose { repair_id: ReconciliationId(245), extent_id: 106 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.886 INFO [2] client ExtentClose { repair_id: ReconciliationId(245), extent_id: 106 }
Sep 22 23:22:18.886 DEBG 245 Close extent 106
Sep 22 23:22:18.886 DEBG 245 Close extent 106
Sep 22 23:22:18.887 DEBG 245 Close extent 106
Sep 22 23:22:18.887 DEBG [2] It's time to notify for 245
Sep 22 23:22:18.887 INFO Completion from [2] id:245 status:true
Sep 22 23:22:18.887 INFO [246/752] Repair commands completed
Sep 22 23:22:18.887 INFO Pop front: ReconcileIO { id: ReconciliationId(246), op: ExtentRepair { repair_id: ReconciliationId(246), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.887 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.887 INFO [0] received reconcile message
Sep 22 23:22:18.887 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(246), op: ExtentRepair { repair_id: ReconciliationId(246), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.887 INFO [0] client ExtentRepair { repair_id: ReconciliationId(246), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.887 INFO [0] Sending repair request ReconciliationId(246)
Sep 22 23:22:18.887 INFO [1] received reconcile message
Sep 22 23:22:18.887 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(246), op: ExtentRepair { repair_id: ReconciliationId(246), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.887 INFO [1] client ExtentRepair { repair_id: ReconciliationId(246), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.887 INFO [1] No action required ReconciliationId(246)
Sep 22 23:22:18.887 INFO [2] received reconcile message
Sep 22 23:22:18.887 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(246), op: ExtentRepair { repair_id: ReconciliationId(246), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.887 INFO [2] client ExtentRepair { repair_id: ReconciliationId(246), extent_id: 106, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.887 INFO [2] No action required ReconciliationId(246)
Sep 22 23:22:18.887 DEBG 246 Repair extent 106 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.888 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/06A.copy"
Sep 22 23:22:18.950 INFO accepted connection, remote_addr: 127.0.0.1:49077, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.951 TRCE incoming request, uri: /extent/106/files, method: GET, req_id: e3f9a519-2f8f-413f-9f62-95e6bbe8cdbc, remote_addr: 127.0.0.1:49077, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.951 INFO request completed, latency_us: 266, response_code: 200, uri: /extent/106/files, method: GET, req_id: e3f9a519-2f8f-413f-9f62-95e6bbe8cdbc, remote_addr: 127.0.0.1:49077, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.951 INFO eid:106 Found repair files: ["06A", "06A.db"]
Sep 22 23:22:18.952 TRCE incoming request, uri: /newextent/106/data, method: GET, req_id: 341f5512-d632-49bf-b29d-4ee34d3de257, remote_addr: 127.0.0.1:49077, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.952 INFO request completed, latency_us: 345, response_code: 200, uri: /newextent/106/data, method: GET, req_id: 341f5512-d632-49bf-b29d-4ee34d3de257, remote_addr: 127.0.0.1:49077, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.957 TRCE incoming request, uri: /newextent/106/db, method: GET, req_id: dfc1fdfd-a3cb-4eff-84c0-712b2881708f, remote_addr: 127.0.0.1:49077, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.957 INFO request completed, latency_us: 295, response_code: 200, uri: /newextent/106/db, method: GET, req_id: dfc1fdfd-a3cb-4eff-84c0-712b2881708f, remote_addr: 127.0.0.1:49077, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:18.958 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/06A.copy" to "/tmp/downstairs-zrMnlo6G/00/000/06A.replace"
Sep 22 23:22:18.958 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.960 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/06A.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.960 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06A"
Sep 22 23:22:18.960 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06A.db"
Sep 22 23:22:18.960 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.960 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/06A.replace" to "/tmp/downstairs-zrMnlo6G/00/000/06A.completed"
Sep 22 23:22:18.960 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.960 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:18.960 DEBG [0] It's time to notify for 246
Sep 22 23:22:18.961 INFO Completion from [0] id:246 status:true
Sep 22 23:22:18.961 INFO [247/752] Repair commands completed
Sep 22 23:22:18.961 INFO Pop front: ReconcileIO { id: ReconciliationId(247), op: ExtentReopen { repair_id: ReconciliationId(247), extent_id: 106 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.961 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.961 INFO [0] received reconcile message
Sep 22 23:22:18.961 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(247), op: ExtentReopen { repair_id: ReconciliationId(247), extent_id: 106 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.961 INFO [0] client ExtentReopen { repair_id: ReconciliationId(247), extent_id: 106 }
Sep 22 23:22:18.961 INFO [1] received reconcile message
Sep 22 23:22:18.961 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(247), op: ExtentReopen { repair_id: ReconciliationId(247), extent_id: 106 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.961 INFO [1] client ExtentReopen { repair_id: ReconciliationId(247), extent_id: 106 }
Sep 22 23:22:18.961 INFO [2] received reconcile message
Sep 22 23:22:18.961 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(247), op: ExtentReopen { repair_id: ReconciliationId(247), extent_id: 106 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.961 INFO [2] client ExtentReopen { repair_id: ReconciliationId(247), extent_id: 106 }
Sep 22 23:22:18.961 DEBG 247 Reopen extent 106
Sep 22 23:22:18.962 DEBG 247 Reopen extent 106
Sep 22 23:22:18.962 DEBG 247 Reopen extent 106
Sep 22 23:22:18.963 DEBG [2] It's time to notify for 247
Sep 22 23:22:18.963 INFO Completion from [2] id:247 status:true
Sep 22 23:22:18.963 INFO [248/752] Repair commands completed
Sep 22 23:22:18.963 INFO Pop front: ReconcileIO { id: ReconciliationId(248), op: ExtentFlush { repair_id: ReconciliationId(248), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.963 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.963 INFO [0] received reconcile message
Sep 22 23:22:18.963 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(248), op: ExtentFlush { repair_id: ReconciliationId(248), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.963 INFO [0] client ExtentFlush { repair_id: ReconciliationId(248), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.963 INFO [1] received reconcile message
Sep 22 23:22:18.963 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(248), op: ExtentFlush { repair_id: ReconciliationId(248), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:18.963 INFO [1] client ExtentFlush { repair_id: ReconciliationId(248), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.963 INFO [2] received reconcile message
Sep 22 23:22:18.963 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(248), op: ExtentFlush { repair_id: ReconciliationId(248), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.963 INFO [2] client ExtentFlush { repair_id: ReconciliationId(248), extent_id: 22, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:18.963 DEBG 248 Flush extent 22 with f:2 g:2
Sep 22 23:22:18.963 DEBG Flush just extent 22 with f:2 and g:2
Sep 22 23:22:18.963 DEBG [1] It's time to notify for 248
Sep 22 23:22:18.963 INFO Completion from [1] id:248 status:true
Sep 22 23:22:18.963 INFO [249/752] Repair commands completed
Sep 22 23:22:18.964 INFO Pop front: ReconcileIO { id: ReconciliationId(249), op: ExtentClose { repair_id: ReconciliationId(249), extent_id: 22 }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.964 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.964 INFO [0] received reconcile message
Sep 22 23:22:18.964 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(249), op: ExtentClose { repair_id: ReconciliationId(249), extent_id: 22 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.964 INFO [0] client ExtentClose { repair_id: ReconciliationId(249), extent_id: 22 }
Sep 22 23:22:18.964 INFO [1] received reconcile message
Sep 22 23:22:18.964 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(249), op: ExtentClose { repair_id: ReconciliationId(249), extent_id: 22 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.964 INFO [1] client ExtentClose { repair_id: ReconciliationId(249), extent_id: 22 }
Sep 22 23:22:18.964 INFO [2] received reconcile message
Sep 22 23:22:18.964 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(249), op: ExtentClose { repair_id: ReconciliationId(249), extent_id: 22 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:18.964 INFO [2] client ExtentClose { repair_id: ReconciliationId(249), extent_id: 22 }
Sep 22 23:22:18.964 DEBG 249 Close extent 22
Sep 22 23:22:18.964 DEBG 249 Close extent 22
Sep 22 23:22:18.964 DEBG 249 Close extent 22
Sep 22 23:22:18.965 DEBG [2] It's time to notify for 249
Sep 22 23:22:18.965 INFO Completion from [2] id:249 status:true
Sep 22 23:22:18.965 INFO [250/752] Repair commands completed
Sep 22 23:22:18.965 INFO Pop front: ReconcileIO { id: ReconciliationId(250), op: ExtentRepair { repair_id: ReconciliationId(250), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:18.965 INFO Sent repair work, now wait for resp
Sep 22 23:22:18.965 INFO [0] received reconcile message
Sep 22 23:22:18.965 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(250), op: ExtentRepair { repair_id: ReconciliationId(250), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:18.965 INFO [0] client ExtentRepair { repair_id: ReconciliationId(250), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.965 INFO [0] Sending repair request ReconciliationId(250)
Sep 22 23:22:18.965 INFO [1] received reconcile message
Sep 22 23:22:18.965 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(250), op: ExtentRepair { repair_id: ReconciliationId(250), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:18.965 INFO [1] client ExtentRepair { repair_id: ReconciliationId(250), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.965 INFO [1] No action required ReconciliationId(250)
Sep 22 23:22:18.965 INFO [2] received reconcile message
Sep 22 23:22:18.965 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(250), op: ExtentRepair { repair_id: ReconciliationId(250), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:18.965 INFO [2] client ExtentRepair { repair_id: ReconciliationId(250), extent_id: 22, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:18.965 INFO [2] No action required ReconciliationId(250)
Sep 22 23:22:18.965 DEBG 250 Repair extent 22 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:18.965 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/016.copy"
Sep 22 23:22:19.031 INFO accepted connection, remote_addr: 127.0.0.1:36746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.032 TRCE incoming request, uri: /extent/22/files, method: GET, req_id: cb1c28eb-5fd9-41cf-b6c1-502363a78462, remote_addr: 127.0.0.1:36746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.032 INFO request completed, latency_us: 272, response_code: 200, uri: /extent/22/files, method: GET, req_id: cb1c28eb-5fd9-41cf-b6c1-502363a78462, remote_addr: 127.0.0.1:36746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.032 INFO eid:22 Found repair files: ["016", "016.db"]
Sep 22 23:22:19.032 TRCE incoming request, uri: /newextent/22/data, method: GET, req_id: 454fa309-5658-43e8-b471-3f2f5f695c08, remote_addr: 127.0.0.1:36746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.033 INFO request completed, latency_us: 347, response_code: 200, uri: /newextent/22/data, method: GET, req_id: 454fa309-5658-43e8-b471-3f2f5f695c08, remote_addr: 127.0.0.1:36746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.038 TRCE incoming request, uri: /newextent/22/db, method: GET, req_id: bc2779bb-8c14-4627-85c3-9fc592adbe5a, remote_addr: 127.0.0.1:36746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.038 INFO request completed, latency_us: 292, response_code: 200, uri: /newextent/22/db, method: GET, req_id: bc2779bb-8c14-4627-85c3-9fc592adbe5a, remote_addr: 127.0.0.1:36746, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.039 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/016.copy" to "/tmp/downstairs-zrMnlo6G/00/000/016.replace"
Sep 22 23:22:19.039 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.041 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/016.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.041 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/016"
Sep 22 23:22:19.041 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/016.db"
Sep 22 23:22:19.041 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.041 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/016.replace" to "/tmp/downstairs-zrMnlo6G/00/000/016.completed"
Sep 22 23:22:19.041 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.041 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.042 DEBG [0] It's time to notify for 250
Sep 22 23:22:19.042 INFO Completion from [0] id:250 status:true
Sep 22 23:22:19.042 INFO [251/752] Repair commands completed
Sep 22 23:22:19.042 INFO Pop front: ReconcileIO { id: ReconciliationId(251), op: ExtentReopen { repair_id: ReconciliationId(251), extent_id: 22 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.042 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.042 INFO [0] received reconcile message
Sep 22 23:22:19.042 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(251), op: ExtentReopen { repair_id: ReconciliationId(251), extent_id: 22 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.042 INFO [0] client ExtentReopen { repair_id: ReconciliationId(251), extent_id: 22 }
Sep 22 23:22:19.042 INFO [1] received reconcile message
Sep 22 23:22:19.042 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(251), op: ExtentReopen { repair_id: ReconciliationId(251), extent_id: 22 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.042 INFO [1] client ExtentReopen { repair_id: ReconciliationId(251), extent_id: 22 }
Sep 22 23:22:19.042 INFO [2] received reconcile message
Sep 22 23:22:19.042 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(251), op: ExtentReopen { repair_id: ReconciliationId(251), extent_id: 22 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.042 INFO [2] client ExtentReopen { repair_id: ReconciliationId(251), extent_id: 22 }
Sep 22 23:22:19.042 DEBG 251 Reopen extent 22
Sep 22 23:22:19.043 DEBG 251 Reopen extent 22
Sep 22 23:22:19.043 DEBG 251 Reopen extent 22
Sep 22 23:22:19.044 DEBG [2] It's time to notify for 251
Sep 22 23:22:19.044 INFO Completion from [2] id:251 status:true
Sep 22 23:22:19.044 INFO [252/752] Repair commands completed
Sep 22 23:22:19.044 INFO Pop front: ReconcileIO { id: ReconciliationId(252), op: ExtentFlush { repair_id: ReconciliationId(252), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.044 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.044 INFO [0] received reconcile message
Sep 22 23:22:19.044 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(252), op: ExtentFlush { repair_id: ReconciliationId(252), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.044 INFO [0] client ExtentFlush { repair_id: ReconciliationId(252), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.044 INFO [1] received reconcile message
Sep 22 23:22:19.044 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(252), op: ExtentFlush { repair_id: ReconciliationId(252), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.044 INFO [1] client ExtentFlush { repair_id: ReconciliationId(252), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.044 INFO [2] received reconcile message
Sep 22 23:22:19.044 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(252), op: ExtentFlush { repair_id: ReconciliationId(252), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.044 INFO [2] client ExtentFlush { repair_id: ReconciliationId(252), extent_id: 146, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.044 DEBG 252 Flush extent 146 with f:2 g:2
Sep 22 23:22:19.044 DEBG Flush just extent 146 with f:2 and g:2
Sep 22 23:22:19.045 DEBG [1] It's time to notify for 252
Sep 22 23:22:19.045 INFO Completion from [1] id:252 status:true
Sep 22 23:22:19.045 INFO [253/752] Repair commands completed
Sep 22 23:22:19.045 INFO Pop front: ReconcileIO { id: ReconciliationId(253), op: ExtentClose { repair_id: ReconciliationId(253), extent_id: 146 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.045 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.045 INFO [0] received reconcile message
Sep 22 23:22:19.045 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(253), op: ExtentClose { repair_id: ReconciliationId(253), extent_id: 146 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.045 INFO [0] client ExtentClose { repair_id: ReconciliationId(253), extent_id: 146 }
Sep 22 23:22:19.045 INFO [1] received reconcile message
Sep 22 23:22:19.045 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(253), op: ExtentClose { repair_id: ReconciliationId(253), extent_id: 146 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.045 INFO [1] client ExtentClose { repair_id: ReconciliationId(253), extent_id: 146 }
Sep 22 23:22:19.045 INFO [2] received reconcile message
Sep 22 23:22:19.045 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(253), op: ExtentClose { repair_id: ReconciliationId(253), extent_id: 146 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.045 INFO [2] client ExtentClose { repair_id: ReconciliationId(253), extent_id: 146 }
Sep 22 23:22:19.045 DEBG 253 Close extent 146
Sep 22 23:22:19.045 DEBG 253 Close extent 146
Sep 22 23:22:19.046 DEBG 253 Close extent 146
Sep 22 23:22:19.046 DEBG [2] It's time to notify for 253
Sep 22 23:22:19.046 INFO Completion from [2] id:253 status:true
Sep 22 23:22:19.046 INFO [254/752] Repair commands completed
Sep 22 23:22:19.046 INFO Pop front: ReconcileIO { id: ReconciliationId(254), op: ExtentRepair { repair_id: ReconciliationId(254), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.046 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.046 INFO [0] received reconcile message
Sep 22 23:22:19.046 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(254), op: ExtentRepair { repair_id: ReconciliationId(254), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.046 INFO [0] client ExtentRepair { repair_id: ReconciliationId(254), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.046 INFO [0] Sending repair request ReconciliationId(254)
Sep 22 23:22:19.046 INFO [1] received reconcile message
Sep 22 23:22:19.046 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(254), op: ExtentRepair { repair_id: ReconciliationId(254), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.046 INFO [1] client ExtentRepair { repair_id: ReconciliationId(254), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.046 INFO [1] No action required ReconciliationId(254)
Sep 22 23:22:19.046 INFO [2] received reconcile message
Sep 22 23:22:19.046 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(254), op: ExtentRepair { repair_id: ReconciliationId(254), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.046 INFO [2] client ExtentRepair { repair_id: ReconciliationId(254), extent_id: 146, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.046 INFO [2] No action required ReconciliationId(254)
Sep 22 23:22:19.047 DEBG 254 Repair extent 146 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.047 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/092.copy"
Sep 22 23:22:19.066 DEBG [0] Read AckReady 1079, : downstairs
Sep 22 23:22:19.067 DEBG up_ds_listen was notified
Sep 22 23:22:19.067 DEBG up_ds_listen process 1079
Sep 22 23:22:19.067 DEBG [A] ack job 1079:80, : downstairs
Sep 22 23:22:19.111 INFO accepted connection, remote_addr: 127.0.0.1:61791, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.111 TRCE incoming request, uri: /extent/146/files, method: GET, req_id: 4c15c710-d209-4122-ac12-fbbd11b79739, remote_addr: 127.0.0.1:61791, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.111 INFO request completed, latency_us: 242, response_code: 200, uri: /extent/146/files, method: GET, req_id: 4c15c710-d209-4122-ac12-fbbd11b79739, remote_addr: 127.0.0.1:61791, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.112 INFO eid:146 Found repair files: ["092", "092.db"]
Sep 22 23:22:19.112 TRCE incoming request, uri: /newextent/146/data, method: GET, req_id: 7756b5db-1e96-4605-9d7d-a5ba68c61622, remote_addr: 127.0.0.1:61791, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.112 INFO request completed, latency_us: 347, response_code: 200, uri: /newextent/146/data, method: GET, req_id: 7756b5db-1e96-4605-9d7d-a5ba68c61622, remote_addr: 127.0.0.1:61791, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.117 TRCE incoming request, uri: /newextent/146/db, method: GET, req_id: 637368e7-b35f-4ffa-86c7-7ad69633481d, remote_addr: 127.0.0.1:61791, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.117 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/146/db, method: GET, req_id: 637368e7-b35f-4ffa-86c7-7ad69633481d, remote_addr: 127.0.0.1:61791, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.118 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/092.copy" to "/tmp/downstairs-zrMnlo6G/00/000/092.replace"
Sep 22 23:22:19.118 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.120 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/092.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.120 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/092"
Sep 22 23:22:19.120 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/092.db"
Sep 22 23:22:19.120 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.120 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/092.replace" to "/tmp/downstairs-zrMnlo6G/00/000/092.completed"
Sep 22 23:22:19.120 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.120 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.120 DEBG [0] It's time to notify for 254
Sep 22 23:22:19.120 INFO Completion from [0] id:254 status:true
Sep 22 23:22:19.120 INFO [255/752] Repair commands completed
Sep 22 23:22:19.120 INFO Pop front: ReconcileIO { id: ReconciliationId(255), op: ExtentReopen { repair_id: ReconciliationId(255), extent_id: 146 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.121 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.121 INFO [0] received reconcile message
Sep 22 23:22:19.121 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(255), op: ExtentReopen { repair_id: ReconciliationId(255), extent_id: 146 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.121 INFO [0] client ExtentReopen { repair_id: ReconciliationId(255), extent_id: 146 }
Sep 22 23:22:19.121 INFO [1] received reconcile message
Sep 22 23:22:19.121 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(255), op: ExtentReopen { repair_id: ReconciliationId(255), extent_id: 146 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.121 INFO [1] client ExtentReopen { repair_id: ReconciliationId(255), extent_id: 146 }
Sep 22 23:22:19.121 INFO [2] received reconcile message
Sep 22 23:22:19.121 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(255), op: ExtentReopen { repair_id: ReconciliationId(255), extent_id: 146 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.121 INFO [2] client ExtentReopen { repair_id: ReconciliationId(255), extent_id: 146 }
Sep 22 23:22:19.121 DEBG 255 Reopen extent 146
Sep 22 23:22:19.121 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:19.122 DEBG 255 Reopen extent 146
Sep 22 23:22:19.122 DEBG 255 Reopen extent 146
Sep 22 23:22:19.123 DEBG [2] It's time to notify for 255
Sep 22 23:22:19.123 INFO Completion from [2] id:255 status:true
Sep 22 23:22:19.123 INFO [256/752] Repair commands completed
Sep 22 23:22:19.123 INFO Pop front: ReconcileIO { id: ReconciliationId(256), op: ExtentFlush { repair_id: ReconciliationId(256), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.123 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.123 INFO [0] received reconcile message
Sep 22 23:22:19.123 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(256), op: ExtentFlush { repair_id: ReconciliationId(256), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.123 INFO [0] client ExtentFlush { repair_id: ReconciliationId(256), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.123 INFO [1] received reconcile message
Sep 22 23:22:19.123 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(256), op: ExtentFlush { repair_id: ReconciliationId(256), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.123 INFO [1] client ExtentFlush { repair_id: ReconciliationId(256), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.123 INFO [2] received reconcile message
Sep 22 23:22:19.123 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(256), op: ExtentFlush { repair_id: ReconciliationId(256), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.123 INFO [2] client ExtentFlush { repair_id: ReconciliationId(256), extent_id: 148, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.123 DEBG 256 Flush extent 148 with f:2 g:2
Sep 22 23:22:19.123 DEBG Flush just extent 148 with f:2 and g:2
Sep 22 23:22:19.123 DEBG [1] It's time to notify for 256
Sep 22 23:22:19.124 INFO Completion from [1] id:256 status:true
Sep 22 23:22:19.124 INFO [257/752] Repair commands completed
Sep 22 23:22:19.124 INFO Pop front: ReconcileIO { id: ReconciliationId(257), op: ExtentClose { repair_id: ReconciliationId(257), extent_id: 148 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.124 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.124 INFO [0] received reconcile message
Sep 22 23:22:19.124 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(257), op: ExtentClose { repair_id: ReconciliationId(257), extent_id: 148 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.124 INFO [0] client ExtentClose { repair_id: ReconciliationId(257), extent_id: 148 }
Sep 22 23:22:19.124 INFO [1] received reconcile message
Sep 22 23:22:19.124 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(257), op: ExtentClose { repair_id: ReconciliationId(257), extent_id: 148 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.124 INFO [1] client ExtentClose { repair_id: ReconciliationId(257), extent_id: 148 }
Sep 22 23:22:19.124 INFO [2] received reconcile message
Sep 22 23:22:19.124 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(257), op: ExtentClose { repair_id: ReconciliationId(257), extent_id: 148 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.124 INFO [2] client ExtentClose { repair_id: ReconciliationId(257), extent_id: 148 }
Sep 22 23:22:19.124 DEBG 257 Close extent 148
Sep 22 23:22:19.124 DEBG 257 Close extent 148
Sep 22 23:22:19.124 DEBG Flush     :1080 extent_limit None deps:[JobId(1079)] res:true f:29 g:1
Sep 22 23:22:19.124 INFO [lossy] sleeping 1 second
Sep 22 23:22:19.125 DEBG 257 Close extent 148
Sep 22 23:22:19.125 DEBG [2] It's time to notify for 257
Sep 22 23:22:19.125 INFO Completion from [2] id:257 status:true
Sep 22 23:22:19.125 INFO [258/752] Repair commands completed
Sep 22 23:22:19.125 INFO Pop front: ReconcileIO { id: ReconciliationId(258), op: ExtentRepair { repair_id: ReconciliationId(258), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.125 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.125 INFO [0] received reconcile message
Sep 22 23:22:19.125 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(258), op: ExtentRepair { repair_id: ReconciliationId(258), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.125 INFO [0] client ExtentRepair { repair_id: ReconciliationId(258), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.125 INFO [0] Sending repair request ReconciliationId(258)
Sep 22 23:22:19.125 INFO [1] received reconcile message
Sep 22 23:22:19.125 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(258), op: ExtentRepair { repair_id: ReconciliationId(258), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.125 INFO [1] client ExtentRepair { repair_id: ReconciliationId(258), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.125 INFO [1] No action required ReconciliationId(258)
Sep 22 23:22:19.125 INFO [2] received reconcile message
Sep 22 23:22:19.125 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(258), op: ExtentRepair { repair_id: ReconciliationId(258), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.125 INFO [2] client ExtentRepair { repair_id: ReconciliationId(258), extent_id: 148, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.125 INFO [2] No action required ReconciliationId(258)
Sep 22 23:22:19.125 DEBG 258 Repair extent 148 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.125 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/094.copy"
Sep 22 23:22:19.130 DEBG Read      :1079 deps:[JobId(1078)] res:true
Sep 22 23:22:19.191 INFO accepted connection, remote_addr: 127.0.0.1:49081, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.191 TRCE incoming request, uri: /extent/148/files, method: GET, req_id: d32eb627-3fa9-492a-843c-9062df76e4fc, remote_addr: 127.0.0.1:49081, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.192 INFO request completed, latency_us: 273, response_code: 200, uri: /extent/148/files, method: GET, req_id: d32eb627-3fa9-492a-843c-9062df76e4fc, remote_addr: 127.0.0.1:49081, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.192 INFO eid:148 Found repair files: ["094", "094.db"]
Sep 22 23:22:19.192 TRCE incoming request, uri: /newextent/148/data, method: GET, req_id: 10d7312d-5180-4f92-9777-bdbcc0b39e5d, remote_addr: 127.0.0.1:49081, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.193 INFO request completed, latency_us: 342, response_code: 200, uri: /newextent/148/data, method: GET, req_id: 10d7312d-5180-4f92-9777-bdbcc0b39e5d, remote_addr: 127.0.0.1:49081, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.197 TRCE incoming request, uri: /newextent/148/db, method: GET, req_id: d8c668cb-ada4-42f1-a1e9-772573ae7ca7, remote_addr: 127.0.0.1:49081, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.198 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/148/db, method: GET, req_id: d8c668cb-ada4-42f1-a1e9-772573ae7ca7, remote_addr: 127.0.0.1:49081, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.199 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/094.copy" to "/tmp/downstairs-zrMnlo6G/00/000/094.replace"
Sep 22 23:22:19.199 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.200 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/094.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.200 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/094"
Sep 22 23:22:19.200 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/094.db"
Sep 22 23:22:19.200 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.200 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/094.replace" to "/tmp/downstairs-zrMnlo6G/00/000/094.completed"
Sep 22 23:22:19.200 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.200 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.201 DEBG [0] It's time to notify for 258
Sep 22 23:22:19.201 INFO Completion from [0] id:258 status:true
Sep 22 23:22:19.201 INFO [259/752] Repair commands completed
Sep 22 23:22:19.201 INFO Pop front: ReconcileIO { id: ReconciliationId(259), op: ExtentReopen { repair_id: ReconciliationId(259), extent_id: 148 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.201 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.201 INFO [0] received reconcile message
Sep 22 23:22:19.201 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(259), op: ExtentReopen { repair_id: ReconciliationId(259), extent_id: 148 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.201 INFO [0] client ExtentReopen { repair_id: ReconciliationId(259), extent_id: 148 }
Sep 22 23:22:19.201 INFO [1] received reconcile message
Sep 22 23:22:19.201 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(259), op: ExtentReopen { repair_id: ReconciliationId(259), extent_id: 148 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.201 INFO [1] client ExtentReopen { repair_id: ReconciliationId(259), extent_id: 148 }
Sep 22 23:22:19.201 INFO [2] received reconcile message
Sep 22 23:22:19.201 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(259), op: ExtentReopen { repair_id: ReconciliationId(259), extent_id: 148 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.201 INFO [2] client ExtentReopen { repair_id: ReconciliationId(259), extent_id: 148 }
Sep 22 23:22:19.201 DEBG 259 Reopen extent 148
Sep 22 23:22:19.202 DEBG 259 Reopen extent 148
Sep 22 23:22:19.202 DEBG IO Read  1081 has deps [JobId(1080)]
Sep 22 23:22:19.203 DEBG 259 Reopen extent 148
Sep 22 23:22:19.203 DEBG [2] It's time to notify for 259
Sep 22 23:22:19.203 INFO Completion from [2] id:259 status:true
Sep 22 23:22:19.203 INFO [260/752] Repair commands completed
Sep 22 23:22:19.203 INFO Pop front: ReconcileIO { id: ReconciliationId(260), op: ExtentFlush { repair_id: ReconciliationId(260), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.203 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.203 INFO [0] received reconcile message
Sep 22 23:22:19.203 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(260), op: ExtentFlush { repair_id: ReconciliationId(260), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.203 INFO [0] client ExtentFlush { repair_id: ReconciliationId(260), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.203 INFO [1] received reconcile message
Sep 22 23:22:19.203 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(260), op: ExtentFlush { repair_id: ReconciliationId(260), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.203 INFO [1] client ExtentFlush { repair_id: ReconciliationId(260), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.204 INFO [2] received reconcile message
Sep 22 23:22:19.204 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(260), op: ExtentFlush { repair_id: ReconciliationId(260), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.204 INFO [2] client ExtentFlush { repair_id: ReconciliationId(260), extent_id: 6, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.204 DEBG 260 Flush extent 6 with f:2 g:2
Sep 22 23:22:19.204 DEBG Flush just extent 6 with f:2 and g:2
Sep 22 23:22:19.204 DEBG [1] It's time to notify for 260
Sep 22 23:22:19.204 INFO Completion from [1] id:260 status:true
Sep 22 23:22:19.204 INFO [261/752] Repair commands completed
Sep 22 23:22:19.204 INFO Pop front: ReconcileIO { id: ReconciliationId(261), op: ExtentClose { repair_id: ReconciliationId(261), extent_id: 6 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.204 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.204 INFO [0] received reconcile message
Sep 22 23:22:19.204 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(261), op: ExtentClose { repair_id: ReconciliationId(261), extent_id: 6 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.204 INFO [0] client ExtentClose { repair_id: ReconciliationId(261), extent_id: 6 }
Sep 22 23:22:19.204 INFO [1] received reconcile message
Sep 22 23:22:19.204 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(261), op: ExtentClose { repair_id: ReconciliationId(261), extent_id: 6 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.204 INFO [1] client ExtentClose { repair_id: ReconciliationId(261), extent_id: 6 }
Sep 22 23:22:19.204 INFO [2] received reconcile message
Sep 22 23:22:19.204 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(261), op: ExtentClose { repair_id: ReconciliationId(261), extent_id: 6 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.204 INFO [2] client ExtentClose { repair_id: ReconciliationId(261), extent_id: 6 }
Sep 22 23:22:19.204 DEBG 261 Close extent 6
Sep 22 23:22:19.205 DEBG 261 Close extent 6
Sep 22 23:22:19.205 DEBG 261 Close extent 6
Sep 22 23:22:19.205 DEBG [2] It's time to notify for 261
Sep 22 23:22:19.205 INFO Completion from [2] id:261 status:true
Sep 22 23:22:19.205 INFO [262/752] Repair commands completed
Sep 22 23:22:19.205 INFO Pop front: ReconcileIO { id: ReconciliationId(262), op: ExtentRepair { repair_id: ReconciliationId(262), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.205 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.205 INFO [0] received reconcile message
Sep 22 23:22:19.205 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(262), op: ExtentRepair { repair_id: ReconciliationId(262), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.205 INFO [0] client ExtentRepair { repair_id: ReconciliationId(262), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.205 INFO [0] Sending repair request ReconciliationId(262)
Sep 22 23:22:19.206 INFO [1] received reconcile message
Sep 22 23:22:19.206 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(262), op: ExtentRepair { repair_id: ReconciliationId(262), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.206 INFO [1] client ExtentRepair { repair_id: ReconciliationId(262), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.206 INFO [1] No action required ReconciliationId(262)
Sep 22 23:22:19.206 INFO [2] received reconcile message
Sep 22 23:22:19.206 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(262), op: ExtentRepair { repair_id: ReconciliationId(262), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.206 INFO [2] client ExtentRepair { repair_id: ReconciliationId(262), extent_id: 6, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.206 INFO [2] No action required ReconciliationId(262)
Sep 22 23:22:19.206 DEBG 262 Repair extent 6 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.206 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/006.copy"
Sep 22 23:22:19.272 INFO accepted connection, remote_addr: 127.0.0.1:58499, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.272 TRCE incoming request, uri: /extent/6/files, method: GET, req_id: 8239d7e2-4d3a-4ed2-a570-c49bda5ed5d6, remote_addr: 127.0.0.1:58499, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.272 INFO request completed, latency_us: 245, response_code: 200, uri: /extent/6/files, method: GET, req_id: 8239d7e2-4d3a-4ed2-a570-c49bda5ed5d6, remote_addr: 127.0.0.1:58499, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.272 INFO eid:6 Found repair files: ["006", "006.db"]
Sep 22 23:22:19.273 TRCE incoming request, uri: /newextent/6/data, method: GET, req_id: 1759023c-0d30-43eb-8d61-895005f53446, remote_addr: 127.0.0.1:58499, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.273 INFO request completed, latency_us: 351, response_code: 200, uri: /newextent/6/data, method: GET, req_id: 1759023c-0d30-43eb-8d61-895005f53446, remote_addr: 127.0.0.1:58499, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.278 TRCE incoming request, uri: /newextent/6/db, method: GET, req_id: 9f529b49-b35c-498c-b4b6-af7d26eba762, remote_addr: 127.0.0.1:58499, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.278 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/6/db, method: GET, req_id: 9f529b49-b35c-498c-b4b6-af7d26eba762, remote_addr: 127.0.0.1:58499, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.279 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/006.copy" to "/tmp/downstairs-zrMnlo6G/00/000/006.replace"
Sep 22 23:22:19.279 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.280 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/006.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.280 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/006"
Sep 22 23:22:19.280 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/006.db"
Sep 22 23:22:19.280 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.280 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/006.replace" to "/tmp/downstairs-zrMnlo6G/00/000/006.completed"
Sep 22 23:22:19.280 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.280 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.281 DEBG [0] It's time to notify for 262
Sep 22 23:22:19.281 INFO Completion from [0] id:262 status:true
Sep 22 23:22:19.281 INFO [263/752] Repair commands completed
Sep 22 23:22:19.281 INFO Pop front: ReconcileIO { id: ReconciliationId(263), op: ExtentReopen { repair_id: ReconciliationId(263), extent_id: 6 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.281 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.281 INFO [0] received reconcile message
Sep 22 23:22:19.281 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(263), op: ExtentReopen { repair_id: ReconciliationId(263), extent_id: 6 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.281 INFO [0] client ExtentReopen { repair_id: ReconciliationId(263), extent_id: 6 }
Sep 22 23:22:19.281 INFO [1] received reconcile message
Sep 22 23:22:19.281 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(263), op: ExtentReopen { repair_id: ReconciliationId(263), extent_id: 6 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.281 INFO [1] client ExtentReopen { repair_id: ReconciliationId(263), extent_id: 6 }
Sep 22 23:22:19.281 INFO [2] received reconcile message
Sep 22 23:22:19.281 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(263), op: ExtentReopen { repair_id: ReconciliationId(263), extent_id: 6 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.281 INFO [2] client ExtentReopen { repair_id: ReconciliationId(263), extent_id: 6 }
Sep 22 23:22:19.281 DEBG 263 Reopen extent 6
Sep 22 23:22:19.282 DEBG 263 Reopen extent 6
Sep 22 23:22:19.282 DEBG 263 Reopen extent 6
Sep 22 23:22:19.283 DEBG [2] It's time to notify for 263
Sep 22 23:22:19.283 INFO Completion from [2] id:263 status:true
Sep 22 23:22:19.283 INFO [264/752] Repair commands completed
Sep 22 23:22:19.283 INFO Pop front: ReconcileIO { id: ReconciliationId(264), op: ExtentFlush { repair_id: ReconciliationId(264), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.283 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.283 INFO [0] received reconcile message
Sep 22 23:22:19.283 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(264), op: ExtentFlush { repair_id: ReconciliationId(264), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.283 INFO [0] client ExtentFlush { repair_id: ReconciliationId(264), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.283 INFO [1] received reconcile message
Sep 22 23:22:19.283 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(264), op: ExtentFlush { repair_id: ReconciliationId(264), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.283 INFO [1] client ExtentFlush { repair_id: ReconciliationId(264), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.283 INFO [2] received reconcile message
Sep 22 23:22:19.283 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(264), op: ExtentFlush { repair_id: ReconciliationId(264), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.283 INFO [2] client ExtentFlush { repair_id: ReconciliationId(264), extent_id: 62, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.284 DEBG 264 Flush extent 62 with f:2 g:2
Sep 22 23:22:19.284 DEBG Flush just extent 62 with f:2 and g:2
Sep 22 23:22:19.284 DEBG [1] It's time to notify for 264
Sep 22 23:22:19.284 INFO Completion from [1] id:264 status:true
Sep 22 23:22:19.284 INFO [265/752] Repair commands completed
Sep 22 23:22:19.284 INFO Pop front: ReconcileIO { id: ReconciliationId(265), op: ExtentClose { repair_id: ReconciliationId(265), extent_id: 62 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.284 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.284 INFO [0] received reconcile message
Sep 22 23:22:19.284 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(265), op: ExtentClose { repair_id: ReconciliationId(265), extent_id: 62 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.284 INFO [0] client ExtentClose { repair_id: ReconciliationId(265), extent_id: 62 }
Sep 22 23:22:19.284 INFO [1] received reconcile message
Sep 22 23:22:19.284 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(265), op: ExtentClose { repair_id: ReconciliationId(265), extent_id: 62 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.284 INFO [1] client ExtentClose { repair_id: ReconciliationId(265), extent_id: 62 }
Sep 22 23:22:19.284 INFO [2] received reconcile message
Sep 22 23:22:19.284 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(265), op: ExtentClose { repair_id: ReconciliationId(265), extent_id: 62 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.284 INFO [2] client ExtentClose { repair_id: ReconciliationId(265), extent_id: 62 }
Sep 22 23:22:19.284 DEBG 265 Close extent 62
Sep 22 23:22:19.284 DEBG 265 Close extent 62
Sep 22 23:22:19.285 DEBG 265 Close extent 62
Sep 22 23:22:19.285 DEBG [2] It's time to notify for 265
Sep 22 23:22:19.285 INFO Completion from [2] id:265 status:true
Sep 22 23:22:19.285 INFO [266/752] Repair commands completed
Sep 22 23:22:19.285 INFO Pop front: ReconcileIO { id: ReconciliationId(266), op: ExtentRepair { repair_id: ReconciliationId(266), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.285 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.285 INFO [0] received reconcile message
Sep 22 23:22:19.285 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(266), op: ExtentRepair { repair_id: ReconciliationId(266), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.285 INFO [0] client ExtentRepair { repair_id: ReconciliationId(266), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.285 INFO [0] Sending repair request ReconciliationId(266)
Sep 22 23:22:19.285 INFO [1] received reconcile message
Sep 22 23:22:19.285 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(266), op: ExtentRepair { repair_id: ReconciliationId(266), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.285 INFO [1] client ExtentRepair { repair_id: ReconciliationId(266), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.285 INFO [1] No action required ReconciliationId(266)
Sep 22 23:22:19.285 INFO [2] received reconcile message
Sep 22 23:22:19.285 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(266), op: ExtentRepair { repair_id: ReconciliationId(266), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.286 INFO [2] client ExtentRepair { repair_id: ReconciliationId(266), extent_id: 62, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.286 INFO [2] No action required ReconciliationId(266)
Sep 22 23:22:19.286 DEBG 266 Repair extent 62 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.286 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/03E.copy"
Sep 22 23:22:19.351 INFO accepted connection, remote_addr: 127.0.0.1:41150, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.352 TRCE incoming request, uri: /extent/62/files, method: GET, req_id: 0b68f468-2861-4eb7-97aa-bed6390fa852, remote_addr: 127.0.0.1:41150, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.352 INFO request completed, latency_us: 243, response_code: 200, uri: /extent/62/files, method: GET, req_id: 0b68f468-2861-4eb7-97aa-bed6390fa852, remote_addr: 127.0.0.1:41150, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.352 INFO eid:62 Found repair files: ["03E", "03E.db"]
Sep 22 23:22:19.353 TRCE incoming request, uri: /newextent/62/data, method: GET, req_id: 3cd6d695-c5d6-4072-b92a-7972a0922a3e, remote_addr: 127.0.0.1:41150, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.353 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/62/data, method: GET, req_id: 3cd6d695-c5d6-4072-b92a-7972a0922a3e, remote_addr: 127.0.0.1:41150, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.358 TRCE incoming request, uri: /newextent/62/db, method: GET, req_id: 16d90d14-230b-4fdc-80e4-653a037e3557, remote_addr: 127.0.0.1:41150, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.358 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/62/db, method: GET, req_id: 16d90d14-230b-4fdc-80e4-653a037e3557, remote_addr: 127.0.0.1:41150, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.359 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/03E.copy" to "/tmp/downstairs-zrMnlo6G/00/000/03E.replace"
Sep 22 23:22:19.359 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.360 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/03E.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.361 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03E"
Sep 22 23:22:19.361 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03E.db"
Sep 22 23:22:19.361 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.361 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/03E.replace" to "/tmp/downstairs-zrMnlo6G/00/000/03E.completed"
Sep 22 23:22:19.361 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.361 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.361 DEBG [0] It's time to notify for 266
Sep 22 23:22:19.361 INFO Completion from [0] id:266 status:true
Sep 22 23:22:19.361 INFO [267/752] Repair commands completed
Sep 22 23:22:19.361 INFO Pop front: ReconcileIO { id: ReconciliationId(267), op: ExtentReopen { repair_id: ReconciliationId(267), extent_id: 62 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.361 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.361 INFO [0] received reconcile message
Sep 22 23:22:19.361 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(267), op: ExtentReopen { repair_id: ReconciliationId(267), extent_id: 62 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.361 INFO [0] client ExtentReopen { repair_id: ReconciliationId(267), extent_id: 62 }
Sep 22 23:22:19.361 INFO [1] received reconcile message
Sep 22 23:22:19.361 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(267), op: ExtentReopen { repair_id: ReconciliationId(267), extent_id: 62 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.361 INFO [1] client ExtentReopen { repair_id: ReconciliationId(267), extent_id: 62 }
Sep 22 23:22:19.361 INFO [2] received reconcile message
Sep 22 23:22:19.362 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(267), op: ExtentReopen { repair_id: ReconciliationId(267), extent_id: 62 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.362 INFO [2] client ExtentReopen { repair_id: ReconciliationId(267), extent_id: 62 }
Sep 22 23:22:19.362 DEBG 267 Reopen extent 62
Sep 22 23:22:19.362 DEBG 267 Reopen extent 62
Sep 22 23:22:19.363 DEBG 267 Reopen extent 62
Sep 22 23:22:19.364 DEBG [2] It's time to notify for 267
Sep 22 23:22:19.364 INFO Completion from [2] id:267 status:true
Sep 22 23:22:19.364 INFO [268/752] Repair commands completed
Sep 22 23:22:19.364 INFO Pop front: ReconcileIO { id: ReconciliationId(268), op: ExtentFlush { repair_id: ReconciliationId(268), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.364 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.364 INFO [0] received reconcile message
Sep 22 23:22:19.364 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(268), op: ExtentFlush { repair_id: ReconciliationId(268), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.364 INFO [0] client ExtentFlush { repair_id: ReconciliationId(268), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.364 INFO [1] received reconcile message
Sep 22 23:22:19.364 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(268), op: ExtentFlush { repair_id: ReconciliationId(268), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.364 INFO [1] client ExtentFlush { repair_id: ReconciliationId(268), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.364 INFO [2] received reconcile message
Sep 22 23:22:19.364 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(268), op: ExtentFlush { repair_id: ReconciliationId(268), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.364 INFO [2] client ExtentFlush { repair_id: ReconciliationId(268), extent_id: 126, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.364 DEBG 268 Flush extent 126 with f:2 g:2
Sep 22 23:22:19.364 DEBG Flush just extent 126 with f:2 and g:2
Sep 22 23:22:19.364 DEBG [1] It's time to notify for 268
Sep 22 23:22:19.364 INFO Completion from [1] id:268 status:true
Sep 22 23:22:19.364 INFO [269/752] Repair commands completed
Sep 22 23:22:19.364 INFO Pop front: ReconcileIO { id: ReconciliationId(269), op: ExtentClose { repair_id: ReconciliationId(269), extent_id: 126 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.364 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.364 INFO [0] received reconcile message
Sep 22 23:22:19.364 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(269), op: ExtentClose { repair_id: ReconciliationId(269), extent_id: 126 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.364 INFO [0] client ExtentClose { repair_id: ReconciliationId(269), extent_id: 126 }
Sep 22 23:22:19.364 INFO [1] received reconcile message
Sep 22 23:22:19.364 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(269), op: ExtentClose { repair_id: ReconciliationId(269), extent_id: 126 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.364 INFO [1] client ExtentClose { repair_id: ReconciliationId(269), extent_id: 126 }
Sep 22 23:22:19.364 INFO [2] received reconcile message
Sep 22 23:22:19.364 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(269), op: ExtentClose { repair_id: ReconciliationId(269), extent_id: 126 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.364 INFO [2] client ExtentClose { repair_id: ReconciliationId(269), extent_id: 126 }
Sep 22 23:22:19.365 DEBG 269 Close extent 126
Sep 22 23:22:19.365 DEBG 269 Close extent 126
Sep 22 23:22:19.365 DEBG 269 Close extent 126
Sep 22 23:22:19.366 DEBG [2] It's time to notify for 269
Sep 22 23:22:19.366 INFO Completion from [2] id:269 status:true
Sep 22 23:22:19.366 INFO [270/752] Repair commands completed
Sep 22 23:22:19.366 INFO Pop front: ReconcileIO { id: ReconciliationId(270), op: ExtentRepair { repair_id: ReconciliationId(270), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.366 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.366 INFO [0] received reconcile message
Sep 22 23:22:19.366 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(270), op: ExtentRepair { repair_id: ReconciliationId(270), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.366 INFO [0] client ExtentRepair { repair_id: ReconciliationId(270), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.366 INFO [0] Sending repair request ReconciliationId(270)
Sep 22 23:22:19.366 INFO [1] received reconcile message
Sep 22 23:22:19.366 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(270), op: ExtentRepair { repair_id: ReconciliationId(270), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.366 INFO [1] client ExtentRepair { repair_id: ReconciliationId(270), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.366 INFO [1] No action required ReconciliationId(270)
Sep 22 23:22:19.366 INFO [2] received reconcile message
Sep 22 23:22:19.366 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(270), op: ExtentRepair { repair_id: ReconciliationId(270), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.366 INFO [2] client ExtentRepair { repair_id: ReconciliationId(270), extent_id: 126, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.366 INFO [2] No action required ReconciliationId(270)
Sep 22 23:22:19.366 DEBG 270 Repair extent 126 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.366 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/07E.copy"
Sep 22 23:22:19.431 INFO accepted connection, remote_addr: 127.0.0.1:53993, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.431 TRCE incoming request, uri: /extent/126/files, method: GET, req_id: 90e158c7-3d63-4551-836d-ec2d05ef5d19, remote_addr: 127.0.0.1:53993, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.432 INFO request completed, latency_us: 279, response_code: 200, uri: /extent/126/files, method: GET, req_id: 90e158c7-3d63-4551-836d-ec2d05ef5d19, remote_addr: 127.0.0.1:53993, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.432 INFO eid:126 Found repair files: ["07E", "07E.db"]
Sep 22 23:22:19.432 TRCE incoming request, uri: /newextent/126/data, method: GET, req_id: d5a8403b-4876-4e36-855c-d6b6a4ffe7dc, remote_addr: 127.0.0.1:53993, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.433 INFO request completed, latency_us: 350, response_code: 200, uri: /newextent/126/data, method: GET, req_id: d5a8403b-4876-4e36-855c-d6b6a4ffe7dc, remote_addr: 127.0.0.1:53993, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.438 TRCE incoming request, uri: /newextent/126/db, method: GET, req_id: bbbcfc5f-4c28-419e-82df-c4d71692afd7, remote_addr: 127.0.0.1:53993, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.438 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/126/db, method: GET, req_id: bbbcfc5f-4c28-419e-82df-c4d71692afd7, remote_addr: 127.0.0.1:53993, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.439 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/07E.copy" to "/tmp/downstairs-zrMnlo6G/00/000/07E.replace"
Sep 22 23:22:19.439 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.440 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/07E.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.440 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07E"
Sep 22 23:22:19.440 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07E.db"
Sep 22 23:22:19.441 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.441 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/07E.replace" to "/tmp/downstairs-zrMnlo6G/00/000/07E.completed"
Sep 22 23:22:19.441 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.441 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.441 DEBG [0] It's time to notify for 270
Sep 22 23:22:19.441 INFO Completion from [0] id:270 status:true
Sep 22 23:22:19.441 INFO [271/752] Repair commands completed
Sep 22 23:22:19.441 INFO Pop front: ReconcileIO { id: ReconciliationId(271), op: ExtentReopen { repair_id: ReconciliationId(271), extent_id: 126 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.441 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.441 INFO [0] received reconcile message
Sep 22 23:22:19.441 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(271), op: ExtentReopen { repair_id: ReconciliationId(271), extent_id: 126 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.441 INFO [0] client ExtentReopen { repair_id: ReconciliationId(271), extent_id: 126 }
Sep 22 23:22:19.441 INFO [1] received reconcile message
Sep 22 23:22:19.441 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(271), op: ExtentReopen { repair_id: ReconciliationId(271), extent_id: 126 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.441 INFO [1] client ExtentReopen { repair_id: ReconciliationId(271), extent_id: 126 }
Sep 22 23:22:19.441 INFO [2] received reconcile message
Sep 22 23:22:19.441 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(271), op: ExtentReopen { repair_id: ReconciliationId(271), extent_id: 126 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.441 INFO [2] client ExtentReopen { repair_id: ReconciliationId(271), extent_id: 126 }
Sep 22 23:22:19.442 DEBG 271 Reopen extent 126
Sep 22 23:22:19.442 DEBG 271 Reopen extent 126
Sep 22 23:22:19.443 DEBG 271 Reopen extent 126
Sep 22 23:22:19.443 DEBG [2] It's time to notify for 271
Sep 22 23:22:19.443 INFO Completion from [2] id:271 status:true
Sep 22 23:22:19.444 INFO [272/752] Repair commands completed
Sep 22 23:22:19.444 INFO Pop front: ReconcileIO { id: ReconciliationId(272), op: ExtentFlush { repair_id: ReconciliationId(272), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.444 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.444 INFO [0] received reconcile message
Sep 22 23:22:19.444 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(272), op: ExtentFlush { repair_id: ReconciliationId(272), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.444 INFO [0] client ExtentFlush { repair_id: ReconciliationId(272), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.444 INFO [1] received reconcile message
Sep 22 23:22:19.444 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(272), op: ExtentFlush { repair_id: ReconciliationId(272), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.444 INFO [1] client ExtentFlush { repair_id: ReconciliationId(272), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.444 INFO [2] received reconcile message
Sep 22 23:22:19.444 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(272), op: ExtentFlush { repair_id: ReconciliationId(272), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.444 INFO [2] client ExtentFlush { repair_id: ReconciliationId(272), extent_id: 128, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.444 DEBG 272 Flush extent 128 with f:2 g:2
Sep 22 23:22:19.444 DEBG Flush just extent 128 with f:2 and g:2
Sep 22 23:22:19.444 DEBG [1] It's time to notify for 272
Sep 22 23:22:19.444 INFO Completion from [1] id:272 status:true
Sep 22 23:22:19.444 INFO [273/752] Repair commands completed
Sep 22 23:22:19.444 INFO Pop front: ReconcileIO { id: ReconciliationId(273), op: ExtentClose { repair_id: ReconciliationId(273), extent_id: 128 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.444 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.444 INFO [0] received reconcile message
Sep 22 23:22:19.444 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(273), op: ExtentClose { repair_id: ReconciliationId(273), extent_id: 128 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.444 INFO [0] client ExtentClose { repair_id: ReconciliationId(273), extent_id: 128 }
Sep 22 23:22:19.444 INFO [1] received reconcile message
Sep 22 23:22:19.444 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(273), op: ExtentClose { repair_id: ReconciliationId(273), extent_id: 128 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.444 INFO [1] client ExtentClose { repair_id: ReconciliationId(273), extent_id: 128 }
Sep 22 23:22:19.444 INFO [2] received reconcile message
Sep 22 23:22:19.444 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(273), op: ExtentClose { repair_id: ReconciliationId(273), extent_id: 128 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.444 INFO [2] client ExtentClose { repair_id: ReconciliationId(273), extent_id: 128 }
Sep 22 23:22:19.445 DEBG 273 Close extent 128
Sep 22 23:22:19.445 DEBG 273 Close extent 128
Sep 22 23:22:19.445 DEBG 273 Close extent 128
Sep 22 23:22:19.445 DEBG [2] It's time to notify for 273
Sep 22 23:22:19.446 INFO Completion from [2] id:273 status:true
Sep 22 23:22:19.446 INFO [274/752] Repair commands completed
Sep 22 23:22:19.446 INFO Pop front: ReconcileIO { id: ReconciliationId(274), op: ExtentRepair { repair_id: ReconciliationId(274), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.446 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.446 INFO [0] received reconcile message
Sep 22 23:22:19.446 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(274), op: ExtentRepair { repair_id: ReconciliationId(274), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.446 INFO [0] client ExtentRepair { repair_id: ReconciliationId(274), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.446 INFO [0] Sending repair request ReconciliationId(274)
Sep 22 23:22:19.446 INFO [1] received reconcile message
Sep 22 23:22:19.446 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(274), op: ExtentRepair { repair_id: ReconciliationId(274), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.446 INFO [1] client ExtentRepair { repair_id: ReconciliationId(274), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.446 INFO [1] No action required ReconciliationId(274)
Sep 22 23:22:19.446 INFO [2] received reconcile message
Sep 22 23:22:19.446 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(274), op: ExtentRepair { repair_id: ReconciliationId(274), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.446 INFO [2] client ExtentRepair { repair_id: ReconciliationId(274), extent_id: 128, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.446 INFO [2] No action required ReconciliationId(274)
Sep 22 23:22:19.446 DEBG 274 Repair extent 128 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.446 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/080.copy"
Sep 22 23:22:19.509 INFO accepted connection, remote_addr: 127.0.0.1:34045, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.509 TRCE incoming request, uri: /extent/128/files, method: GET, req_id: 8f0487c8-02c7-4061-81fe-73f9be8c1301, remote_addr: 127.0.0.1:34045, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.509 INFO request completed, latency_us: 205, response_code: 200, uri: /extent/128/files, method: GET, req_id: 8f0487c8-02c7-4061-81fe-73f9be8c1301, remote_addr: 127.0.0.1:34045, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.510 INFO eid:128 Found repair files: ["080", "080.db"]
Sep 22 23:22:19.510 TRCE incoming request, uri: /newextent/128/data, method: GET, req_id: 1bb08b04-43fc-493f-8838-265a061e1485, remote_addr: 127.0.0.1:34045, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.510 INFO request completed, latency_us: 322, response_code: 200, uri: /newextent/128/data, method: GET, req_id: 1bb08b04-43fc-493f-8838-265a061e1485, remote_addr: 127.0.0.1:34045, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.515 TRCE incoming request, uri: /newextent/128/db, method: GET, req_id: 35292bfb-7665-493d-84a4-8c15b437983e, remote_addr: 127.0.0.1:34045, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.515 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/128/db, method: GET, req_id: 35292bfb-7665-493d-84a4-8c15b437983e, remote_addr: 127.0.0.1:34045, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.516 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/080.copy" to "/tmp/downstairs-zrMnlo6G/00/000/080.replace"
Sep 22 23:22:19.516 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.517 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/080.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.517 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/080"
Sep 22 23:22:19.518 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/080.db"
Sep 22 23:22:19.518 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.518 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/080.replace" to "/tmp/downstairs-zrMnlo6G/00/000/080.completed"
Sep 22 23:22:19.518 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.518 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.518 DEBG [0] It's time to notify for 274
Sep 22 23:22:19.518 INFO Completion from [0] id:274 status:true
Sep 22 23:22:19.518 INFO [275/752] Repair commands completed
Sep 22 23:22:19.518 INFO Pop front: ReconcileIO { id: ReconciliationId(275), op: ExtentReopen { repair_id: ReconciliationId(275), extent_id: 128 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.518 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.518 INFO [0] received reconcile message
Sep 22 23:22:19.518 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(275), op: ExtentReopen { repair_id: ReconciliationId(275), extent_id: 128 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.518 INFO [0] client ExtentReopen { repair_id: ReconciliationId(275), extent_id: 128 }
Sep 22 23:22:19.518 INFO [1] received reconcile message
Sep 22 23:22:19.518 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(275), op: ExtentReopen { repair_id: ReconciliationId(275), extent_id: 128 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.518 INFO [1] client ExtentReopen { repair_id: ReconciliationId(275), extent_id: 128 }
Sep 22 23:22:19.518 INFO [2] received reconcile message
Sep 22 23:22:19.518 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(275), op: ExtentReopen { repair_id: ReconciliationId(275), extent_id: 128 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.518 INFO [2] client ExtentReopen { repair_id: ReconciliationId(275), extent_id: 128 }
Sep 22 23:22:19.518 DEBG 275 Reopen extent 128
Sep 22 23:22:19.519 DEBG 275 Reopen extent 128
Sep 22 23:22:19.520 DEBG 275 Reopen extent 128
Sep 22 23:22:19.520 DEBG [2] It's time to notify for 275
Sep 22 23:22:19.520 INFO Completion from [2] id:275 status:true
Sep 22 23:22:19.520 INFO [276/752] Repair commands completed
Sep 22 23:22:19.520 INFO Pop front: ReconcileIO { id: ReconciliationId(276), op: ExtentFlush { repair_id: ReconciliationId(276), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.520 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.520 INFO [0] received reconcile message
Sep 22 23:22:19.520 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(276), op: ExtentFlush { repair_id: ReconciliationId(276), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.520 INFO [0] client ExtentFlush { repair_id: ReconciliationId(276), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.520 INFO [1] received reconcile message
Sep 22 23:22:19.520 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(276), op: ExtentFlush { repair_id: ReconciliationId(276), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.520 INFO [1] client ExtentFlush { repair_id: ReconciliationId(276), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.520 INFO [2] received reconcile message
Sep 22 23:22:19.520 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(276), op: ExtentFlush { repair_id: ReconciliationId(276), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.520 INFO [2] client ExtentFlush { repair_id: ReconciliationId(276), extent_id: 79, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.521 DEBG 276 Flush extent 79 with f:2 g:2
Sep 22 23:22:19.521 DEBG Flush just extent 79 with f:2 and g:2
Sep 22 23:22:19.521 DEBG [1] It's time to notify for 276
Sep 22 23:22:19.521 INFO Completion from [1] id:276 status:true
Sep 22 23:22:19.521 INFO [277/752] Repair commands completed
Sep 22 23:22:19.521 INFO Pop front: ReconcileIO { id: ReconciliationId(277), op: ExtentClose { repair_id: ReconciliationId(277), extent_id: 79 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.521 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.521 INFO [0] received reconcile message
Sep 22 23:22:19.521 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(277), op: ExtentClose { repair_id: ReconciliationId(277), extent_id: 79 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.521 INFO [0] client ExtentClose { repair_id: ReconciliationId(277), extent_id: 79 }
Sep 22 23:22:19.521 INFO [1] received reconcile message
Sep 22 23:22:19.521 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(277), op: ExtentClose { repair_id: ReconciliationId(277), extent_id: 79 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.521 INFO [1] client ExtentClose { repair_id: ReconciliationId(277), extent_id: 79 }
Sep 22 23:22:19.521 INFO [2] received reconcile message
Sep 22 23:22:19.521 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(277), op: ExtentClose { repair_id: ReconciliationId(277), extent_id: 79 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.521 INFO [2] client ExtentClose { repair_id: ReconciliationId(277), extent_id: 79 }
Sep 22 23:22:19.521 DEBG 277 Close extent 79
Sep 22 23:22:19.521 DEBG 277 Close extent 79
Sep 22 23:22:19.522 DEBG 277 Close extent 79
Sep 22 23:22:19.522 DEBG [2] It's time to notify for 277
Sep 22 23:22:19.522 INFO Completion from [2] id:277 status:true
Sep 22 23:22:19.522 INFO [278/752] Repair commands completed
Sep 22 23:22:19.522 INFO Pop front: ReconcileIO { id: ReconciliationId(278), op: ExtentRepair { repair_id: ReconciliationId(278), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.522 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.522 INFO [0] received reconcile message
Sep 22 23:22:19.522 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(278), op: ExtentRepair { repair_id: ReconciliationId(278), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.522 INFO [0] client ExtentRepair { repair_id: ReconciliationId(278), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.522 INFO [0] Sending repair request ReconciliationId(278)
Sep 22 23:22:19.522 INFO [1] received reconcile message
Sep 22 23:22:19.522 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(278), op: ExtentRepair { repair_id: ReconciliationId(278), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.522 INFO [1] client ExtentRepair { repair_id: ReconciliationId(278), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.522 INFO [1] No action required ReconciliationId(278)
Sep 22 23:22:19.522 INFO [2] received reconcile message
Sep 22 23:22:19.522 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(278), op: ExtentRepair { repair_id: ReconciliationId(278), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.523 INFO [2] client ExtentRepair { repair_id: ReconciliationId(278), extent_id: 79, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.523 INFO [2] No action required ReconciliationId(278)
Sep 22 23:22:19.523 DEBG 278 Repair extent 79 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.523 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/04F.copy"
Sep 22 23:22:19.554 DEBG Flush     :1080 extent_limit None deps:[JobId(1079)] res:true f:29 g:1
Sep 22 23:22:19.554 WARN returning error on read!
Sep 22 23:22:19.554 DEBG Read      :1081 deps:[JobId(1080)] res:false
Sep 22 23:22:19.554 INFO [lossy] skipping 1081
Sep 22 23:22:19.560 DEBG Read      :1081 deps:[JobId(1080)] res:true
Sep 22 23:22:19.588 INFO accepted connection, remote_addr: 127.0.0.1:58831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.588 TRCE incoming request, uri: /extent/79/files, method: GET, req_id: 21002347-7169-4518-adb7-7fc33fe36e1a, remote_addr: 127.0.0.1:58831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.589 INFO request completed, latency_us: 218, response_code: 200, uri: /extent/79/files, method: GET, req_id: 21002347-7169-4518-adb7-7fc33fe36e1a, remote_addr: 127.0.0.1:58831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.589 INFO eid:79 Found repair files: ["04F", "04F.db"]
Sep 22 23:22:19.589 TRCE incoming request, uri: /newextent/79/data, method: GET, req_id: 5dd0bccf-b48d-4f76-984e-c1f7cd3f45fb, remote_addr: 127.0.0.1:58831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.589 INFO request completed, latency_us: 333, response_code: 200, uri: /newextent/79/data, method: GET, req_id: 5dd0bccf-b48d-4f76-984e-c1f7cd3f45fb, remote_addr: 127.0.0.1:58831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.594 TRCE incoming request, uri: /newextent/79/db, method: GET, req_id: 640a7219-d062-4d35-aa7a-6b066d3adce7, remote_addr: 127.0.0.1:58831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.594 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/79/db, method: GET, req_id: 640a7219-d062-4d35-aa7a-6b066d3adce7, remote_addr: 127.0.0.1:58831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.596 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/04F.copy" to "/tmp/downstairs-zrMnlo6G/00/000/04F.replace"
Sep 22 23:22:19.596 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.596 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/04F.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.597 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04F"
Sep 22 23:22:19.597 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04F.db"
Sep 22 23:22:19.597 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.597 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/04F.replace" to "/tmp/downstairs-zrMnlo6G/00/000/04F.completed"
Sep 22 23:22:19.597 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.597 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.597 DEBG [0] It's time to notify for 278
Sep 22 23:22:19.597 INFO Completion from [0] id:278 status:true
Sep 22 23:22:19.597 INFO [279/752] Repair commands completed
Sep 22 23:22:19.597 INFO Pop front: ReconcileIO { id: ReconciliationId(279), op: ExtentReopen { repair_id: ReconciliationId(279), extent_id: 79 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.597 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.597 INFO [0] received reconcile message
Sep 22 23:22:19.597 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(279), op: ExtentReopen { repair_id: ReconciliationId(279), extent_id: 79 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.597 INFO [0] client ExtentReopen { repair_id: ReconciliationId(279), extent_id: 79 }
Sep 22 23:22:19.597 INFO [1] received reconcile message
Sep 22 23:22:19.597 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(279), op: ExtentReopen { repair_id: ReconciliationId(279), extent_id: 79 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.597 INFO [1] client ExtentReopen { repair_id: ReconciliationId(279), extent_id: 79 }
Sep 22 23:22:19.598 INFO [2] received reconcile message
Sep 22 23:22:19.598 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(279), op: ExtentReopen { repair_id: ReconciliationId(279), extent_id: 79 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.598 INFO [2] client ExtentReopen { repair_id: ReconciliationId(279), extent_id: 79 }
Sep 22 23:22:19.598 DEBG 279 Reopen extent 79
Sep 22 23:22:19.598 DEBG 279 Reopen extent 79
Sep 22 23:22:19.599 DEBG 279 Reopen extent 79
Sep 22 23:22:19.599 DEBG [2] It's time to notify for 279
Sep 22 23:22:19.600 INFO Completion from [2] id:279 status:true
Sep 22 23:22:19.600 INFO [280/752] Repair commands completed
Sep 22 23:22:19.600 INFO Pop front: ReconcileIO { id: ReconciliationId(280), op: ExtentFlush { repair_id: ReconciliationId(280), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.600 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.600 INFO [0] received reconcile message
Sep 22 23:22:19.600 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(280), op: ExtentFlush { repair_id: ReconciliationId(280), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.600 INFO [0] client ExtentFlush { repair_id: ReconciliationId(280), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.600 INFO [1] received reconcile message
Sep 22 23:22:19.600 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(280), op: ExtentFlush { repair_id: ReconciliationId(280), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.600 INFO [1] client ExtentFlush { repair_id: ReconciliationId(280), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.600 INFO [2] received reconcile message
Sep 22 23:22:19.600 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(280), op: ExtentFlush { repair_id: ReconciliationId(280), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.600 INFO [2] client ExtentFlush { repair_id: ReconciliationId(280), extent_id: 144, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.600 DEBG 280 Flush extent 144 with f:2 g:2
Sep 22 23:22:19.600 DEBG Flush just extent 144 with f:2 and g:2
Sep 22 23:22:19.600 DEBG [1] It's time to notify for 280
Sep 22 23:22:19.600 INFO Completion from [1] id:280 status:true
Sep 22 23:22:19.600 INFO [281/752] Repair commands completed
Sep 22 23:22:19.600 INFO Pop front: ReconcileIO { id: ReconciliationId(281), op: ExtentClose { repair_id: ReconciliationId(281), extent_id: 144 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.600 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.600 INFO [0] received reconcile message
Sep 22 23:22:19.600 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(281), op: ExtentClose { repair_id: ReconciliationId(281), extent_id: 144 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.600 INFO [0] client ExtentClose { repair_id: ReconciliationId(281), extent_id: 144 }
Sep 22 23:22:19.600 INFO [1] received reconcile message
Sep 22 23:22:19.600 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(281), op: ExtentClose { repair_id: ReconciliationId(281), extent_id: 144 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.600 INFO [1] client ExtentClose { repair_id: ReconciliationId(281), extent_id: 144 }
Sep 22 23:22:19.600 INFO [2] received reconcile message
Sep 22 23:22:19.600 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(281), op: ExtentClose { repair_id: ReconciliationId(281), extent_id: 144 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.600 INFO [2] client ExtentClose { repair_id: ReconciliationId(281), extent_id: 144 }
Sep 22 23:22:19.601 DEBG 281 Close extent 144
Sep 22 23:22:19.601 DEBG 281 Close extent 144
Sep 22 23:22:19.601 DEBG 281 Close extent 144
Sep 22 23:22:19.601 DEBG [2] It's time to notify for 281
Sep 22 23:22:19.602 INFO Completion from [2] id:281 status:true
Sep 22 23:22:19.602 INFO [282/752] Repair commands completed
Sep 22 23:22:19.602 INFO Pop front: ReconcileIO { id: ReconciliationId(282), op: ExtentRepair { repair_id: ReconciliationId(282), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.602 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.602 INFO [0] received reconcile message
Sep 22 23:22:19.602 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(282), op: ExtentRepair { repair_id: ReconciliationId(282), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.602 INFO [0] client ExtentRepair { repair_id: ReconciliationId(282), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.602 INFO [0] Sending repair request ReconciliationId(282)
Sep 22 23:22:19.602 INFO [1] received reconcile message
Sep 22 23:22:19.602 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(282), op: ExtentRepair { repair_id: ReconciliationId(282), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.602 INFO [1] client ExtentRepair { repair_id: ReconciliationId(282), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.602 INFO [1] No action required ReconciliationId(282)
Sep 22 23:22:19.602 INFO [2] received reconcile message
Sep 22 23:22:19.602 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(282), op: ExtentRepair { repair_id: ReconciliationId(282), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.602 INFO [2] client ExtentRepair { repair_id: ReconciliationId(282), extent_id: 144, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.602 INFO [2] No action required ReconciliationId(282)
Sep 22 23:22:19.602 DEBG 282 Repair extent 144 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.602 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/090.copy"
Sep 22 23:22:19.667 INFO accepted connection, remote_addr: 127.0.0.1:43851, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.668 TRCE incoming request, uri: /extent/144/files, method: GET, req_id: f2d721e2-54aa-4595-8ead-eec597fb89d2, remote_addr: 127.0.0.1:43851, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.668 INFO request completed, latency_us: 269, response_code: 200, uri: /extent/144/files, method: GET, req_id: f2d721e2-54aa-4595-8ead-eec597fb89d2, remote_addr: 127.0.0.1:43851, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.668 INFO eid:144 Found repair files: ["090", "090.db"]
Sep 22 23:22:19.669 TRCE incoming request, uri: /newextent/144/data, method: GET, req_id: 0d4b3c41-c732-4673-a782-763384d3e8fb, remote_addr: 127.0.0.1:43851, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.669 INFO request completed, latency_us: 346, response_code: 200, uri: /newextent/144/data, method: GET, req_id: 0d4b3c41-c732-4673-a782-763384d3e8fb, remote_addr: 127.0.0.1:43851, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.674 TRCE incoming request, uri: /newextent/144/db, method: GET, req_id: 566c0aaf-33e1-4e2b-b6d6-4b5e611e224a, remote_addr: 127.0.0.1:43851, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.674 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/144/db, method: GET, req_id: 566c0aaf-33e1-4e2b-b6d6-4b5e611e224a, remote_addr: 127.0.0.1:43851, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.675 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/090.copy" to "/tmp/downstairs-zrMnlo6G/00/000/090.replace"
Sep 22 23:22:19.675 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.676 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/090.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.677 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/090"
Sep 22 23:22:19.677 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/090.db"
Sep 22 23:22:19.677 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.677 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/090.replace" to "/tmp/downstairs-zrMnlo6G/00/000/090.completed"
Sep 22 23:22:19.677 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.677 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.677 DEBG [0] It's time to notify for 282
Sep 22 23:22:19.677 INFO Completion from [0] id:282 status:true
Sep 22 23:22:19.677 INFO [283/752] Repair commands completed
Sep 22 23:22:19.677 INFO Pop front: ReconcileIO { id: ReconciliationId(283), op: ExtentReopen { repair_id: ReconciliationId(283), extent_id: 144 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.677 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.677 INFO [0] received reconcile message
Sep 22 23:22:19.677 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(283), op: ExtentReopen { repair_id: ReconciliationId(283), extent_id: 144 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.677 INFO [0] client ExtentReopen { repair_id: ReconciliationId(283), extent_id: 144 }
Sep 22 23:22:19.677 INFO [1] received reconcile message
Sep 22 23:22:19.677 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(283), op: ExtentReopen { repair_id: ReconciliationId(283), extent_id: 144 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.678 INFO [1] client ExtentReopen { repair_id: ReconciliationId(283), extent_id: 144 }
Sep 22 23:22:19.678 INFO [2] received reconcile message
Sep 22 23:22:19.678 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(283), op: ExtentReopen { repair_id: ReconciliationId(283), extent_id: 144 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.678 INFO [2] client ExtentReopen { repair_id: ReconciliationId(283), extent_id: 144 }
Sep 22 23:22:19.678 DEBG 283 Reopen extent 144
Sep 22 23:22:19.679 DEBG 283 Reopen extent 144
Sep 22 23:22:19.679 DEBG 283 Reopen extent 144
Sep 22 23:22:19.680 DEBG [2] It's time to notify for 283
Sep 22 23:22:19.680 INFO Completion from [2] id:283 status:true
Sep 22 23:22:19.680 INFO [284/752] Repair commands completed
Sep 22 23:22:19.680 INFO Pop front: ReconcileIO { id: ReconciliationId(284), op: ExtentFlush { repair_id: ReconciliationId(284), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.680 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.680 INFO [0] received reconcile message
Sep 22 23:22:19.680 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(284), op: ExtentFlush { repair_id: ReconciliationId(284), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.680 INFO [0] client ExtentFlush { repair_id: ReconciliationId(284), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.680 INFO [1] received reconcile message
Sep 22 23:22:19.680 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(284), op: ExtentFlush { repair_id: ReconciliationId(284), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.680 INFO [1] client ExtentFlush { repair_id: ReconciliationId(284), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.680 INFO [2] received reconcile message
Sep 22 23:22:19.680 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(284), op: ExtentFlush { repair_id: ReconciliationId(284), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.680 INFO [2] client ExtentFlush { repair_id: ReconciliationId(284), extent_id: 160, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.680 DEBG 284 Flush extent 160 with f:2 g:2
Sep 22 23:22:19.680 DEBG Flush just extent 160 with f:2 and g:2
Sep 22 23:22:19.680 DEBG [1] It's time to notify for 284
Sep 22 23:22:19.680 INFO Completion from [1] id:284 status:true
Sep 22 23:22:19.680 INFO [285/752] Repair commands completed
Sep 22 23:22:19.680 INFO Pop front: ReconcileIO { id: ReconciliationId(285), op: ExtentClose { repair_id: ReconciliationId(285), extent_id: 160 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.680 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.680 INFO [0] received reconcile message
Sep 22 23:22:19.680 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(285), op: ExtentClose { repair_id: ReconciliationId(285), extent_id: 160 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.680 INFO [0] client ExtentClose { repair_id: ReconciliationId(285), extent_id: 160 }
Sep 22 23:22:19.680 INFO [1] received reconcile message
Sep 22 23:22:19.681 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(285), op: ExtentClose { repair_id: ReconciliationId(285), extent_id: 160 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.681 INFO [1] client ExtentClose { repair_id: ReconciliationId(285), extent_id: 160 }
Sep 22 23:22:19.681 INFO [2] received reconcile message
Sep 22 23:22:19.681 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(285), op: ExtentClose { repair_id: ReconciliationId(285), extent_id: 160 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.681 INFO [2] client ExtentClose { repair_id: ReconciliationId(285), extent_id: 160 }
Sep 22 23:22:19.681 DEBG 285 Close extent 160
Sep 22 23:22:19.681 DEBG 285 Close extent 160
Sep 22 23:22:19.681 DEBG 285 Close extent 160
Sep 22 23:22:19.682 DEBG [2] It's time to notify for 285
Sep 22 23:22:19.682 INFO Completion from [2] id:285 status:true
Sep 22 23:22:19.682 INFO [286/752] Repair commands completed
Sep 22 23:22:19.682 INFO Pop front: ReconcileIO { id: ReconciliationId(286), op: ExtentRepair { repair_id: ReconciliationId(286), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.682 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.682 INFO [0] received reconcile message
Sep 22 23:22:19.682 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(286), op: ExtentRepair { repair_id: ReconciliationId(286), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.682 INFO [0] client ExtentRepair { repair_id: ReconciliationId(286), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.682 INFO [0] Sending repair request ReconciliationId(286)
Sep 22 23:22:19.682 INFO [1] received reconcile message
Sep 22 23:22:19.682 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(286), op: ExtentRepair { repair_id: ReconciliationId(286), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.682 INFO [1] client ExtentRepair { repair_id: ReconciliationId(286), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.682 INFO [1] No action required ReconciliationId(286)
Sep 22 23:22:19.682 INFO [2] received reconcile message
Sep 22 23:22:19.682 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(286), op: ExtentRepair { repair_id: ReconciliationId(286), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.682 INFO [2] client ExtentRepair { repair_id: ReconciliationId(286), extent_id: 160, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.682 INFO [2] No action required ReconciliationId(286)
Sep 22 23:22:19.682 DEBG 286 Repair extent 160 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.682 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0A0.copy"
Sep 22 23:22:19.747 INFO accepted connection, remote_addr: 127.0.0.1:39291, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.747 TRCE incoming request, uri: /extent/160/files, method: GET, req_id: ea967da3-493c-4d19-9205-0bf359070c4a, remote_addr: 127.0.0.1:39291, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.747 INFO request completed, latency_us: 228, response_code: 200, uri: /extent/160/files, method: GET, req_id: ea967da3-493c-4d19-9205-0bf359070c4a, remote_addr: 127.0.0.1:39291, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.748 INFO eid:160 Found repair files: ["0A0", "0A0.db"]
Sep 22 23:22:19.748 TRCE incoming request, uri: /newextent/160/data, method: GET, req_id: 3eb0cf72-349e-4474-bae4-6852bddfc97f, remote_addr: 127.0.0.1:39291, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.748 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/160/data, method: GET, req_id: 3eb0cf72-349e-4474-bae4-6852bddfc97f, remote_addr: 127.0.0.1:39291, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.753 TRCE incoming request, uri: /newextent/160/db, method: GET, req_id: b8b8aaf1-33eb-4aa9-a42c-1f6ebb450da5, remote_addr: 127.0.0.1:39291, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.753 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/160/db, method: GET, req_id: b8b8aaf1-33eb-4aa9-a42c-1f6ebb450da5, remote_addr: 127.0.0.1:39291, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.754 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0A0.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0A0.replace"
Sep 22 23:22:19.754 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.755 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0A0.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.755 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A0"
Sep 22 23:22:19.755 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A0.db"
Sep 22 23:22:19.756 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.756 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0A0.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0A0.completed"
Sep 22 23:22:19.756 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.756 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.756 DEBG [0] It's time to notify for 286
Sep 22 23:22:19.756 INFO Completion from [0] id:286 status:true
Sep 22 23:22:19.756 INFO [287/752] Repair commands completed
Sep 22 23:22:19.756 INFO Pop front: ReconcileIO { id: ReconciliationId(287), op: ExtentReopen { repair_id: ReconciliationId(287), extent_id: 160 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.756 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.756 INFO [0] received reconcile message
Sep 22 23:22:19.756 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(287), op: ExtentReopen { repair_id: ReconciliationId(287), extent_id: 160 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.756 INFO [0] client ExtentReopen { repair_id: ReconciliationId(287), extent_id: 160 }
Sep 22 23:22:19.756 INFO [1] received reconcile message
Sep 22 23:22:19.756 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(287), op: ExtentReopen { repair_id: ReconciliationId(287), extent_id: 160 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.756 INFO [1] client ExtentReopen { repair_id: ReconciliationId(287), extent_id: 160 }
Sep 22 23:22:19.756 INFO [2] received reconcile message
Sep 22 23:22:19.756 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(287), op: ExtentReopen { repair_id: ReconciliationId(287), extent_id: 160 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.756 INFO [2] client ExtentReopen { repair_id: ReconciliationId(287), extent_id: 160 }
Sep 22 23:22:19.756 DEBG 287 Reopen extent 160
Sep 22 23:22:19.757 DEBG 287 Reopen extent 160
Sep 22 23:22:19.758 DEBG 287 Reopen extent 160
Sep 22 23:22:19.758 DEBG [2] It's time to notify for 287
Sep 22 23:22:19.758 INFO Completion from [2] id:287 status:true
Sep 22 23:22:19.758 INFO [288/752] Repair commands completed
Sep 22 23:22:19.758 INFO Pop front: ReconcileIO { id: ReconciliationId(288), op: ExtentFlush { repair_id: ReconciliationId(288), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.758 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.758 INFO [0] received reconcile message
Sep 22 23:22:19.758 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(288), op: ExtentFlush { repair_id: ReconciliationId(288), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.758 INFO [0] client ExtentFlush { repair_id: ReconciliationId(288), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.758 INFO [1] received reconcile message
Sep 22 23:22:19.758 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(288), op: ExtentFlush { repair_id: ReconciliationId(288), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.758 INFO [1] client ExtentFlush { repair_id: ReconciliationId(288), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.758 INFO [2] received reconcile message
Sep 22 23:22:19.759 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(288), op: ExtentFlush { repair_id: ReconciliationId(288), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.759 INFO [2] client ExtentFlush { repair_id: ReconciliationId(288), extent_id: 133, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.759 DEBG 288 Flush extent 133 with f:2 g:2
Sep 22 23:22:19.759 DEBG Flush just extent 133 with f:2 and g:2
Sep 22 23:22:19.759 DEBG [1] It's time to notify for 288
Sep 22 23:22:19.759 INFO Completion from [1] id:288 status:true
Sep 22 23:22:19.759 INFO [289/752] Repair commands completed
Sep 22 23:22:19.759 INFO Pop front: ReconcileIO { id: ReconciliationId(289), op: ExtentClose { repair_id: ReconciliationId(289), extent_id: 133 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.759 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.759 INFO [0] received reconcile message
Sep 22 23:22:19.759 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(289), op: ExtentClose { repair_id: ReconciliationId(289), extent_id: 133 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.759 INFO [0] client ExtentClose { repair_id: ReconciliationId(289), extent_id: 133 }
Sep 22 23:22:19.759 INFO [1] received reconcile message
Sep 22 23:22:19.759 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(289), op: ExtentClose { repair_id: ReconciliationId(289), extent_id: 133 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.759 INFO [1] client ExtentClose { repair_id: ReconciliationId(289), extent_id: 133 }
Sep 22 23:22:19.759 INFO [2] received reconcile message
Sep 22 23:22:19.759 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(289), op: ExtentClose { repair_id: ReconciliationId(289), extent_id: 133 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.759 INFO [2] client ExtentClose { repair_id: ReconciliationId(289), extent_id: 133 }
Sep 22 23:22:19.759 DEBG 289 Close extent 133
Sep 22 23:22:19.759 DEBG 289 Close extent 133
Sep 22 23:22:19.760 DEBG 289 Close extent 133
Sep 22 23:22:19.760 DEBG [2] It's time to notify for 289
Sep 22 23:22:19.760 INFO Completion from [2] id:289 status:true
Sep 22 23:22:19.760 INFO [290/752] Repair commands completed
Sep 22 23:22:19.760 INFO Pop front: ReconcileIO { id: ReconciliationId(290), op: ExtentRepair { repair_id: ReconciliationId(290), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.760 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.760 INFO [0] received reconcile message
Sep 22 23:22:19.760 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(290), op: ExtentRepair { repair_id: ReconciliationId(290), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.760 INFO [0] client ExtentRepair { repair_id: ReconciliationId(290), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.760 INFO [0] Sending repair request ReconciliationId(290)
Sep 22 23:22:19.760 INFO [1] received reconcile message
Sep 22 23:22:19.761 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(290), op: ExtentRepair { repair_id: ReconciliationId(290), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.761 INFO [1] client ExtentRepair { repair_id: ReconciliationId(290), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.761 INFO [1] No action required ReconciliationId(290)
Sep 22 23:22:19.761 INFO [2] received reconcile message
Sep 22 23:22:19.761 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(290), op: ExtentRepair { repair_id: ReconciliationId(290), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.761 INFO [2] client ExtentRepair { repair_id: ReconciliationId(290), extent_id: 133, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.761 INFO [2] No action required ReconciliationId(290)
Sep 22 23:22:19.761 DEBG 290 Repair extent 133 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.761 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/085.copy"
Sep 22 23:22:19.826 INFO accepted connection, remote_addr: 127.0.0.1:53931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.826 TRCE incoming request, uri: /extent/133/files, method: GET, req_id: 31f2709e-fd5e-41fa-b695-fb7b0a2368c3, remote_addr: 127.0.0.1:53931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.827 INFO request completed, latency_us: 280, response_code: 200, uri: /extent/133/files, method: GET, req_id: 31f2709e-fd5e-41fa-b695-fb7b0a2368c3, remote_addr: 127.0.0.1:53931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.827 INFO eid:133 Found repair files: ["085", "085.db"]
Sep 22 23:22:19.827 TRCE incoming request, uri: /newextent/133/data, method: GET, req_id: 2d9ada90-b46a-48ce-a439-431ced8e6874, remote_addr: 127.0.0.1:53931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.828 INFO request completed, latency_us: 350, response_code: 200, uri: /newextent/133/data, method: GET, req_id: 2d9ada90-b46a-48ce-a439-431ced8e6874, remote_addr: 127.0.0.1:53931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.833 TRCE incoming request, uri: /newextent/133/db, method: GET, req_id: a557e406-b269-4b71-ad24-7c069cefcb9e, remote_addr: 127.0.0.1:53931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.833 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/133/db, method: GET, req_id: a557e406-b269-4b71-ad24-7c069cefcb9e, remote_addr: 127.0.0.1:53931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.834 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/085.copy" to "/tmp/downstairs-zrMnlo6G/00/000/085.replace"
Sep 22 23:22:19.834 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.835 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/085.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.836 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/085"
Sep 22 23:22:19.836 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/085.db"
Sep 22 23:22:19.836 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.836 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/085.replace" to "/tmp/downstairs-zrMnlo6G/00/000/085.completed"
Sep 22 23:22:19.836 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.836 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.836 DEBG [0] It's time to notify for 290
Sep 22 23:22:19.836 INFO Completion from [0] id:290 status:true
Sep 22 23:22:19.836 INFO [291/752] Repair commands completed
Sep 22 23:22:19.836 INFO Pop front: ReconcileIO { id: ReconciliationId(291), op: ExtentReopen { repair_id: ReconciliationId(291), extent_id: 133 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.836 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.836 INFO [0] received reconcile message
Sep 22 23:22:19.836 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(291), op: ExtentReopen { repair_id: ReconciliationId(291), extent_id: 133 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.837 INFO [0] client ExtentReopen { repair_id: ReconciliationId(291), extent_id: 133 }
Sep 22 23:22:19.837 INFO [1] received reconcile message
Sep 22 23:22:19.837 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(291), op: ExtentReopen { repair_id: ReconciliationId(291), extent_id: 133 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.837 INFO [1] client ExtentReopen { repair_id: ReconciliationId(291), extent_id: 133 }
Sep 22 23:22:19.837 INFO [2] received reconcile message
Sep 22 23:22:19.837 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(291), op: ExtentReopen { repair_id: ReconciliationId(291), extent_id: 133 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.837 INFO [2] client ExtentReopen { repair_id: ReconciliationId(291), extent_id: 133 }
Sep 22 23:22:19.837 DEBG 291 Reopen extent 133
Sep 22 23:22:19.838 DEBG 291 Reopen extent 133
Sep 22 23:22:19.838 DEBG 291 Reopen extent 133
Sep 22 23:22:19.839 DEBG [2] It's time to notify for 291
Sep 22 23:22:19.839 INFO Completion from [2] id:291 status:true
Sep 22 23:22:19.839 INFO [292/752] Repair commands completed
Sep 22 23:22:19.839 INFO Pop front: ReconcileIO { id: ReconciliationId(292), op: ExtentFlush { repair_id: ReconciliationId(292), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.839 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.839 INFO [0] received reconcile message
Sep 22 23:22:19.839 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(292), op: ExtentFlush { repair_id: ReconciliationId(292), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.839 INFO [0] client ExtentFlush { repair_id: ReconciliationId(292), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.839 INFO [1] received reconcile message
Sep 22 23:22:19.839 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(292), op: ExtentFlush { repair_id: ReconciliationId(292), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.839 INFO [1] client ExtentFlush { repair_id: ReconciliationId(292), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.839 INFO [2] received reconcile message
Sep 22 23:22:19.839 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(292), op: ExtentFlush { repair_id: ReconciliationId(292), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.839 INFO [2] client ExtentFlush { repair_id: ReconciliationId(292), extent_id: 109, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.839 DEBG 292 Flush extent 109 with f:2 g:2
Sep 22 23:22:19.839 DEBG Flush just extent 109 with f:2 and g:2
Sep 22 23:22:19.839 DEBG [1] It's time to notify for 292
Sep 22 23:22:19.839 INFO Completion from [1] id:292 status:true
Sep 22 23:22:19.839 INFO [293/752] Repair commands completed
Sep 22 23:22:19.839 INFO Pop front: ReconcileIO { id: ReconciliationId(293), op: ExtentClose { repair_id: ReconciliationId(293), extent_id: 109 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.840 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.840 INFO [0] received reconcile message
Sep 22 23:22:19.840 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(293), op: ExtentClose { repair_id: ReconciliationId(293), extent_id: 109 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.840 INFO [0] client ExtentClose { repair_id: ReconciliationId(293), extent_id: 109 }
Sep 22 23:22:19.840 INFO [1] received reconcile message
Sep 22 23:22:19.840 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(293), op: ExtentClose { repair_id: ReconciliationId(293), extent_id: 109 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.840 INFO [1] client ExtentClose { repair_id: ReconciliationId(293), extent_id: 109 }
Sep 22 23:22:19.840 INFO [2] received reconcile message
Sep 22 23:22:19.840 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(293), op: ExtentClose { repair_id: ReconciliationId(293), extent_id: 109 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.840 INFO [2] client ExtentClose { repair_id: ReconciliationId(293), extent_id: 109 }
Sep 22 23:22:19.840 DEBG 293 Close extent 109
Sep 22 23:22:19.840 DEBG 293 Close extent 109
Sep 22 23:22:19.840 DEBG 293 Close extent 109
Sep 22 23:22:19.841 DEBG [2] It's time to notify for 293
Sep 22 23:22:19.841 INFO Completion from [2] id:293 status:true
Sep 22 23:22:19.841 INFO [294/752] Repair commands completed
Sep 22 23:22:19.841 INFO Pop front: ReconcileIO { id: ReconciliationId(294), op: ExtentRepair { repair_id: ReconciliationId(294), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.841 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.841 INFO [0] received reconcile message
Sep 22 23:22:19.841 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(294), op: ExtentRepair { repair_id: ReconciliationId(294), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.841 INFO [0] client ExtentRepair { repair_id: ReconciliationId(294), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.841 INFO [0] Sending repair request ReconciliationId(294)
Sep 22 23:22:19.841 INFO [1] received reconcile message
Sep 22 23:22:19.841 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(294), op: ExtentRepair { repair_id: ReconciliationId(294), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.841 INFO [1] client ExtentRepair { repair_id: ReconciliationId(294), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.841 INFO [1] No action required ReconciliationId(294)
Sep 22 23:22:19.841 INFO [2] received reconcile message
Sep 22 23:22:19.841 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(294), op: ExtentRepair { repair_id: ReconciliationId(294), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.841 INFO [2] client ExtentRepair { repair_id: ReconciliationId(294), extent_id: 109, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.841 INFO [2] No action required ReconciliationId(294)
Sep 22 23:22:19.841 DEBG 294 Repair extent 109 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.841 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/06D.copy"
Sep 22 23:22:19.907 INFO accepted connection, remote_addr: 127.0.0.1:58922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.907 TRCE incoming request, uri: /extent/109/files, method: GET, req_id: cab63320-c4de-4799-9ae4-e02afc363b39, remote_addr: 127.0.0.1:58922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.908 INFO request completed, latency_us: 271, response_code: 200, uri: /extent/109/files, method: GET, req_id: cab63320-c4de-4799-9ae4-e02afc363b39, remote_addr: 127.0.0.1:58922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.908 INFO eid:109 Found repair files: ["06D", "06D.db"]
Sep 22 23:22:19.908 TRCE incoming request, uri: /newextent/109/data, method: GET, req_id: fe230adc-3caf-4113-885b-7d993f337372, remote_addr: 127.0.0.1:58922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.909 INFO request completed, latency_us: 362, response_code: 200, uri: /newextent/109/data, method: GET, req_id: fe230adc-3caf-4113-885b-7d993f337372, remote_addr: 127.0.0.1:58922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.913 TRCE incoming request, uri: /newextent/109/db, method: GET, req_id: 2f8650df-e693-4ba3-9af2-b043a4160cbd, remote_addr: 127.0.0.1:58922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.913 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/109/db, method: GET, req_id: 2f8650df-e693-4ba3-9af2-b043a4160cbd, remote_addr: 127.0.0.1:58922, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.915 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/06D.copy" to "/tmp/downstairs-zrMnlo6G/00/000/06D.replace"
Sep 22 23:22:19.915 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.916 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/06D.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.916 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06D"
Sep 22 23:22:19.916 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06D.db"
Sep 22 23:22:19.916 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.916 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/06D.replace" to "/tmp/downstairs-zrMnlo6G/00/000/06D.completed"
Sep 22 23:22:19.916 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.916 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.916 DEBG [0] It's time to notify for 294
Sep 22 23:22:19.916 INFO Completion from [0] id:294 status:true
Sep 22 23:22:19.916 INFO [295/752] Repair commands completed
Sep 22 23:22:19.917 INFO Pop front: ReconcileIO { id: ReconciliationId(295), op: ExtentReopen { repair_id: ReconciliationId(295), extent_id: 109 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.917 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.917 INFO [0] received reconcile message
Sep 22 23:22:19.917 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(295), op: ExtentReopen { repair_id: ReconciliationId(295), extent_id: 109 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.917 INFO [0] client ExtentReopen { repair_id: ReconciliationId(295), extent_id: 109 }
Sep 22 23:22:19.917 INFO [1] received reconcile message
Sep 22 23:22:19.917 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(295), op: ExtentReopen { repair_id: ReconciliationId(295), extent_id: 109 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.917 INFO [1] client ExtentReopen { repair_id: ReconciliationId(295), extent_id: 109 }
Sep 22 23:22:19.917 INFO [2] received reconcile message
Sep 22 23:22:19.917 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(295), op: ExtentReopen { repair_id: ReconciliationId(295), extent_id: 109 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.917 INFO [2] client ExtentReopen { repair_id: ReconciliationId(295), extent_id: 109 }
Sep 22 23:22:19.917 DEBG 295 Reopen extent 109
Sep 22 23:22:19.918 DEBG 295 Reopen extent 109
Sep 22 23:22:19.918 DEBG 295 Reopen extent 109
Sep 22 23:22:19.919 DEBG [2] It's time to notify for 295
Sep 22 23:22:19.919 INFO Completion from [2] id:295 status:true
Sep 22 23:22:19.919 INFO [296/752] Repair commands completed
Sep 22 23:22:19.919 INFO Pop front: ReconcileIO { id: ReconciliationId(296), op: ExtentFlush { repair_id: ReconciliationId(296), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.919 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.919 INFO [0] received reconcile message
Sep 22 23:22:19.919 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(296), op: ExtentFlush { repair_id: ReconciliationId(296), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.919 INFO [0] client ExtentFlush { repair_id: ReconciliationId(296), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.919 INFO [1] received reconcile message
Sep 22 23:22:19.919 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(296), op: ExtentFlush { repair_id: ReconciliationId(296), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.919 INFO [1] client ExtentFlush { repair_id: ReconciliationId(296), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.919 INFO [2] received reconcile message
Sep 22 23:22:19.919 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(296), op: ExtentFlush { repair_id: ReconciliationId(296), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.919 INFO [2] client ExtentFlush { repair_id: ReconciliationId(296), extent_id: 169, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.919 DEBG 296 Flush extent 169 with f:2 g:2
Sep 22 23:22:19.919 DEBG Flush just extent 169 with f:2 and g:2
Sep 22 23:22:19.919 DEBG [1] It's time to notify for 296
Sep 22 23:22:19.920 INFO Completion from [1] id:296 status:true
Sep 22 23:22:19.920 INFO [297/752] Repair commands completed
Sep 22 23:22:19.920 INFO Pop front: ReconcileIO { id: ReconciliationId(297), op: ExtentClose { repair_id: ReconciliationId(297), extent_id: 169 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.920 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.920 INFO [0] received reconcile message
Sep 22 23:22:19.920 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(297), op: ExtentClose { repair_id: ReconciliationId(297), extent_id: 169 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.920 INFO [0] client ExtentClose { repair_id: ReconciliationId(297), extent_id: 169 }
Sep 22 23:22:19.920 INFO [1] received reconcile message
Sep 22 23:22:19.920 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(297), op: ExtentClose { repair_id: ReconciliationId(297), extent_id: 169 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.920 INFO [1] client ExtentClose { repair_id: ReconciliationId(297), extent_id: 169 }
Sep 22 23:22:19.920 INFO [2] received reconcile message
Sep 22 23:22:19.920 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(297), op: ExtentClose { repair_id: ReconciliationId(297), extent_id: 169 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.920 INFO [2] client ExtentClose { repair_id: ReconciliationId(297), extent_id: 169 }
Sep 22 23:22:19.920 DEBG 297 Close extent 169
Sep 22 23:22:19.920 DEBG 297 Close extent 169
Sep 22 23:22:19.921 DEBG 297 Close extent 169
Sep 22 23:22:19.921 DEBG [2] It's time to notify for 297
Sep 22 23:22:19.921 INFO Completion from [2] id:297 status:true
Sep 22 23:22:19.921 INFO [298/752] Repair commands completed
Sep 22 23:22:19.921 INFO Pop front: ReconcileIO { id: ReconciliationId(298), op: ExtentRepair { repair_id: ReconciliationId(298), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.921 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.921 INFO [0] received reconcile message
Sep 22 23:22:19.921 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(298), op: ExtentRepair { repair_id: ReconciliationId(298), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.921 INFO [0] client ExtentRepair { repair_id: ReconciliationId(298), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.921 INFO [0] Sending repair request ReconciliationId(298)
Sep 22 23:22:19.921 INFO [1] received reconcile message
Sep 22 23:22:19.921 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(298), op: ExtentRepair { repair_id: ReconciliationId(298), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.921 INFO [1] client ExtentRepair { repair_id: ReconciliationId(298), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.921 INFO [1] No action required ReconciliationId(298)
Sep 22 23:22:19.921 INFO [2] received reconcile message
Sep 22 23:22:19.921 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(298), op: ExtentRepair { repair_id: ReconciliationId(298), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:19.921 INFO [2] client ExtentRepair { repair_id: ReconciliationId(298), extent_id: 169, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:19.921 INFO [2] No action required ReconciliationId(298)
Sep 22 23:22:19.921 DEBG 298 Repair extent 169 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:19.922 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0A9.copy"
Sep 22 23:22:19.964 ERRO [1] job id 1081 saw error GenericError("test error")
Sep 22 23:22:19.964 DEBG up_ds_listen was notified
Sep 22 23:22:19.964 DEBG up_ds_listen process 1080
Sep 22 23:22:19.964 DEBG [A] ack job 1080:81, : downstairs
Sep 22 23:22:19.965 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:19.965 DEBG IO Flush 1082 has deps [JobId(1081), JobId(1080)]
Sep 22 23:22:19.965 DEBG Flush     :1080 extent_limit None deps:[JobId(1079)] res:true f:29 g:1
Sep 22 23:22:19.971 DEBG Read      :1081 deps:[JobId(1080)] res:true
Sep 22 23:22:19.986 INFO accepted connection, remote_addr: 127.0.0.1:37931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.986 TRCE incoming request, uri: /extent/169/files, method: GET, req_id: 02e0c845-3775-41fc-9e32-c55861482a62, remote_addr: 127.0.0.1:37931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.986 INFO request completed, latency_us: 218, response_code: 200, uri: /extent/169/files, method: GET, req_id: 02e0c845-3775-41fc-9e32-c55861482a62, remote_addr: 127.0.0.1:37931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.986 INFO eid:169 Found repair files: ["0A9", "0A9.db"]
Sep 22 23:22:19.987 TRCE incoming request, uri: /newextent/169/data, method: GET, req_id: 823ca92e-6b90-4e96-b095-ae476b5bd6a8, remote_addr: 127.0.0.1:37931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.987 INFO request completed, latency_us: 265, response_code: 200, uri: /newextent/169/data, method: GET, req_id: 823ca92e-6b90-4e96-b095-ae476b5bd6a8, remote_addr: 127.0.0.1:37931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.992 TRCE incoming request, uri: /newextent/169/db, method: GET, req_id: 36d05e91-b798-45b9-bfaf-f37ed4e8c023, remote_addr: 127.0.0.1:37931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.992 DEBG [rc] retire 1080 clears [JobId(1079), JobId(1080)], : downstairs
Sep 22 23:22:19.992 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/169/db, method: GET, req_id: 36d05e91-b798-45b9-bfaf-f37ed4e8c023, remote_addr: 127.0.0.1:37931, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:19.994 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0A9.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0A9.replace"
Sep 22 23:22:19.994 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.994 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0A9.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.995 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A9"
Sep 22 23:22:19.995 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A9.db"
Sep 22 23:22:19.995 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.995 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0A9.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0A9.completed"
Sep 22 23:22:19.995 WARN returning error on flush!
Sep 22 23:22:19.995 DEBG Flush     :1082 extent_limit None deps:[JobId(1081), JobId(1080)] res:false f:30 g:1
Sep 22 23:22:19.995 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.995 DEBG Flush     :1082 extent_limit None deps:[JobId(1081), JobId(1080)] res:true f:30 g:1
Sep 22 23:22:19.995 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:19.995 INFO [lossy] sleeping 1 second
Sep 22 23:22:19.995 DEBG [0] It's time to notify for 298
Sep 22 23:22:19.995 INFO Completion from [0] id:298 status:true
Sep 22 23:22:19.995 INFO [299/752] Repair commands completed
Sep 22 23:22:19.995 INFO Pop front: ReconcileIO { id: ReconciliationId(299), op: ExtentReopen { repair_id: ReconciliationId(299), extent_id: 169 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.995 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.995 INFO [0] received reconcile message
Sep 22 23:22:19.995 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(299), op: ExtentReopen { repair_id: ReconciliationId(299), extent_id: 169 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.995 INFO [0] client ExtentReopen { repair_id: ReconciliationId(299), extent_id: 169 }
Sep 22 23:22:19.995 INFO [1] received reconcile message
Sep 22 23:22:19.996 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(299), op: ExtentReopen { repair_id: ReconciliationId(299), extent_id: 169 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.996 INFO [1] client ExtentReopen { repair_id: ReconciliationId(299), extent_id: 169 }
Sep 22 23:22:19.996 INFO [2] received reconcile message
Sep 22 23:22:19.996 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(299), op: ExtentReopen { repair_id: ReconciliationId(299), extent_id: 169 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.996 INFO [2] client ExtentReopen { repair_id: ReconciliationId(299), extent_id: 169 }
Sep 22 23:22:19.996 DEBG 299 Reopen extent 169
Sep 22 23:22:19.996 DEBG 299 Reopen extent 169
Sep 22 23:22:19.997 DEBG 299 Reopen extent 169
Sep 22 23:22:19.997 DEBG [2] It's time to notify for 299
Sep 22 23:22:19.998 INFO Completion from [2] id:299 status:true
Sep 22 23:22:19.998 INFO [300/752] Repair commands completed
Sep 22 23:22:19.998 INFO Pop front: ReconcileIO { id: ReconciliationId(300), op: ExtentFlush { repair_id: ReconciliationId(300), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.998 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.998 INFO [0] received reconcile message
Sep 22 23:22:19.998 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(300), op: ExtentFlush { repair_id: ReconciliationId(300), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.998 INFO [0] client ExtentFlush { repair_id: ReconciliationId(300), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.998 INFO [1] received reconcile message
Sep 22 23:22:19.998 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(300), op: ExtentFlush { repair_id: ReconciliationId(300), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:19.998 INFO [1] client ExtentFlush { repair_id: ReconciliationId(300), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.998 INFO [2] received reconcile message
Sep 22 23:22:19.998 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(300), op: ExtentFlush { repair_id: ReconciliationId(300), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.998 INFO [2] client ExtentFlush { repair_id: ReconciliationId(300), extent_id: 28, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:19.998 DEBG 300 Flush extent 28 with f:2 g:2
Sep 22 23:22:19.998 DEBG Flush just extent 28 with f:2 and g:2
Sep 22 23:22:19.998 DEBG [1] It's time to notify for 300
Sep 22 23:22:19.998 INFO Completion from [1] id:300 status:true
Sep 22 23:22:19.998 INFO [301/752] Repair commands completed
Sep 22 23:22:19.998 INFO Pop front: ReconcileIO { id: ReconciliationId(301), op: ExtentClose { repair_id: ReconciliationId(301), extent_id: 28 }, state: ClientData([New, New, New]) }
Sep 22 23:22:19.998 INFO Sent repair work, now wait for resp
Sep 22 23:22:19.998 INFO [0] received reconcile message
Sep 22 23:22:19.998 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(301), op: ExtentClose { repair_id: ReconciliationId(301), extent_id: 28 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:19.998 INFO [0] client ExtentClose { repair_id: ReconciliationId(301), extent_id: 28 }
Sep 22 23:22:19.998 INFO [1] received reconcile message
Sep 22 23:22:19.998 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(301), op: ExtentClose { repair_id: ReconciliationId(301), extent_id: 28 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:19.998 INFO [1] client ExtentClose { repair_id: ReconciliationId(301), extent_id: 28 }
Sep 22 23:22:19.998 INFO [2] received reconcile message
Sep 22 23:22:19.998 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(301), op: ExtentClose { repair_id: ReconciliationId(301), extent_id: 28 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:19.998 INFO [2] client ExtentClose { repair_id: ReconciliationId(301), extent_id: 28 }
Sep 22 23:22:19.999 DEBG 301 Close extent 28
Sep 22 23:22:19.999 DEBG 301 Close extent 28
Sep 22 23:22:19.999 DEBG 301 Close extent 28
Sep 22 23:22:19.999 DEBG [2] It's time to notify for 301
Sep 22 23:22:20.000 INFO Completion from [2] id:301 status:true
Sep 22 23:22:20.000 INFO [302/752] Repair commands completed
Sep 22 23:22:20.000 INFO Pop front: ReconcileIO { id: ReconciliationId(302), op: ExtentRepair { repair_id: ReconciliationId(302), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.000 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.000 INFO [0] received reconcile message
Sep 22 23:22:20.000 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(302), op: ExtentRepair { repair_id: ReconciliationId(302), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.000 INFO [0] client ExtentRepair { repair_id: ReconciliationId(302), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.000 INFO [0] Sending repair request ReconciliationId(302)
Sep 22 23:22:20.000 INFO [1] received reconcile message
Sep 22 23:22:20.000 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(302), op: ExtentRepair { repair_id: ReconciliationId(302), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.000 INFO [1] client ExtentRepair { repair_id: ReconciliationId(302), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.000 INFO [1] No action required ReconciliationId(302)
Sep 22 23:22:20.000 INFO [2] received reconcile message
Sep 22 23:22:20.000 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(302), op: ExtentRepair { repair_id: ReconciliationId(302), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.000 INFO [2] client ExtentRepair { repair_id: ReconciliationId(302), extent_id: 28, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.000 INFO [2] No action required ReconciliationId(302)
Sep 22 23:22:20.000 DEBG 302 Repair extent 28 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.000 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/01C.copy"
Sep 22 23:22:20.066 INFO accepted connection, remote_addr: 127.0.0.1:58112, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.067 TRCE incoming request, uri: /extent/28/files, method: GET, req_id: fd5c7a80-db50-4bae-bb66-709a50ef7f23, remote_addr: 127.0.0.1:58112, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.067 INFO request completed, latency_us: 266, response_code: 200, uri: /extent/28/files, method: GET, req_id: fd5c7a80-db50-4bae-bb66-709a50ef7f23, remote_addr: 127.0.0.1:58112, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.067 INFO eid:28 Found repair files: ["01C", "01C.db"]
Sep 22 23:22:20.068 TRCE incoming request, uri: /newextent/28/data, method: GET, req_id: 234fbddd-f924-41f2-abc3-901b792ba330, remote_addr: 127.0.0.1:58112, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.068 INFO request completed, latency_us: 344, response_code: 200, uri: /newextent/28/data, method: GET, req_id: 234fbddd-f924-41f2-abc3-901b792ba330, remote_addr: 127.0.0.1:58112, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.073 TRCE incoming request, uri: /newextent/28/db, method: GET, req_id: 14f6e5e4-e941-4f1c-890b-a73e984ddb46, remote_addr: 127.0.0.1:58112, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.073 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/28/db, method: GET, req_id: 14f6e5e4-e941-4f1c-890b-a73e984ddb46, remote_addr: 127.0.0.1:58112, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.074 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/01C.copy" to "/tmp/downstairs-zrMnlo6G/00/000/01C.replace"
Sep 22 23:22:20.074 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.075 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/01C.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.075 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01C"
Sep 22 23:22:20.075 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01C.db"
Sep 22 23:22:20.075 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.075 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/01C.replace" to "/tmp/downstairs-zrMnlo6G/00/000/01C.completed"
Sep 22 23:22:20.075 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.075 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.076 DEBG [0] It's time to notify for 302
Sep 22 23:22:20.076 INFO Completion from [0] id:302 status:true
Sep 22 23:22:20.076 INFO [303/752] Repair commands completed
Sep 22 23:22:20.076 INFO Pop front: ReconcileIO { id: ReconciliationId(303), op: ExtentReopen { repair_id: ReconciliationId(303), extent_id: 28 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.076 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.076 INFO [0] received reconcile message
Sep 22 23:22:20.076 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(303), op: ExtentReopen { repair_id: ReconciliationId(303), extent_id: 28 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.076 INFO [0] client ExtentReopen { repair_id: ReconciliationId(303), extent_id: 28 }
Sep 22 23:22:20.076 INFO [1] received reconcile message
Sep 22 23:22:20.076 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(303), op: ExtentReopen { repair_id: ReconciliationId(303), extent_id: 28 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.076 INFO [1] client ExtentReopen { repair_id: ReconciliationId(303), extent_id: 28 }
Sep 22 23:22:20.076 INFO [2] received reconcile message
Sep 22 23:22:20.076 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(303), op: ExtentReopen { repair_id: ReconciliationId(303), extent_id: 28 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.076 INFO [2] client ExtentReopen { repair_id: ReconciliationId(303), extent_id: 28 }
Sep 22 23:22:20.076 DEBG 303 Reopen extent 28
Sep 22 23:22:20.077 DEBG 303 Reopen extent 28
Sep 22 23:22:20.077 DEBG 303 Reopen extent 28
Sep 22 23:22:20.078 DEBG [2] It's time to notify for 303
Sep 22 23:22:20.078 INFO Completion from [2] id:303 status:true
Sep 22 23:22:20.078 INFO [304/752] Repair commands completed
Sep 22 23:22:20.078 INFO Pop front: ReconcileIO { id: ReconciliationId(304), op: ExtentFlush { repair_id: ReconciliationId(304), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.078 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.078 INFO [0] received reconcile message
Sep 22 23:22:20.078 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(304), op: ExtentFlush { repair_id: ReconciliationId(304), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.078 INFO [0] client ExtentFlush { repair_id: ReconciliationId(304), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.078 INFO [1] received reconcile message
Sep 22 23:22:20.078 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(304), op: ExtentFlush { repair_id: ReconciliationId(304), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.078 INFO [1] client ExtentFlush { repair_id: ReconciliationId(304), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.078 INFO [2] received reconcile message
Sep 22 23:22:20.078 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(304), op: ExtentFlush { repair_id: ReconciliationId(304), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.078 INFO [2] client ExtentFlush { repair_id: ReconciliationId(304), extent_id: 49, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.079 DEBG 304 Flush extent 49 with f:2 g:2
Sep 22 23:22:20.079 DEBG Flush just extent 49 with f:2 and g:2
Sep 22 23:22:20.079 DEBG [1] It's time to notify for 304
Sep 22 23:22:20.079 INFO Completion from [1] id:304 status:true
Sep 22 23:22:20.079 INFO [305/752] Repair commands completed
Sep 22 23:22:20.079 INFO Pop front: ReconcileIO { id: ReconciliationId(305), op: ExtentClose { repair_id: ReconciliationId(305), extent_id: 49 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.079 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.079 INFO [0] received reconcile message
Sep 22 23:22:20.079 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(305), op: ExtentClose { repair_id: ReconciliationId(305), extent_id: 49 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.079 INFO [0] client ExtentClose { repair_id: ReconciliationId(305), extent_id: 49 }
Sep 22 23:22:20.079 INFO [1] received reconcile message
Sep 22 23:22:20.079 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(305), op: ExtentClose { repair_id: ReconciliationId(305), extent_id: 49 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.079 INFO [1] client ExtentClose { repair_id: ReconciliationId(305), extent_id: 49 }
Sep 22 23:22:20.079 INFO [2] received reconcile message
Sep 22 23:22:20.079 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(305), op: ExtentClose { repair_id: ReconciliationId(305), extent_id: 49 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.079 INFO [2] client ExtentClose { repair_id: ReconciliationId(305), extent_id: 49 }
Sep 22 23:22:20.079 DEBG 305 Close extent 49
Sep 22 23:22:20.079 DEBG 305 Close extent 49
Sep 22 23:22:20.080 DEBG 305 Close extent 49
Sep 22 23:22:20.080 DEBG [2] It's time to notify for 305
Sep 22 23:22:20.080 INFO Completion from [2] id:305 status:true
Sep 22 23:22:20.080 INFO [306/752] Repair commands completed
Sep 22 23:22:20.080 INFO Pop front: ReconcileIO { id: ReconciliationId(306), op: ExtentRepair { repair_id: ReconciliationId(306), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.080 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.080 INFO [0] received reconcile message
Sep 22 23:22:20.080 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(306), op: ExtentRepair { repair_id: ReconciliationId(306), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.080 INFO [0] client ExtentRepair { repair_id: ReconciliationId(306), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.080 INFO [0] Sending repair request ReconciliationId(306)
Sep 22 23:22:20.080 INFO [1] received reconcile message
Sep 22 23:22:20.080 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(306), op: ExtentRepair { repair_id: ReconciliationId(306), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.080 INFO [1] client ExtentRepair { repair_id: ReconciliationId(306), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.080 INFO [1] No action required ReconciliationId(306)
Sep 22 23:22:20.080 INFO [2] received reconcile message
Sep 22 23:22:20.080 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(306), op: ExtentRepair { repair_id: ReconciliationId(306), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.080 INFO [2] client ExtentRepair { repair_id: ReconciliationId(306), extent_id: 49, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.081 INFO [2] No action required ReconciliationId(306)
Sep 22 23:22:20.081 DEBG 306 Repair extent 49 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.081 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/031.copy"
Sep 22 23:22:20.145 INFO accepted connection, remote_addr: 127.0.0.1:43274, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.146 TRCE incoming request, uri: /extent/49/files, method: GET, req_id: 31305576-2012-450d-9de0-fad4a24c3c0a, remote_addr: 127.0.0.1:43274, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.146 INFO request completed, latency_us: 305, response_code: 200, uri: /extent/49/files, method: GET, req_id: 31305576-2012-450d-9de0-fad4a24c3c0a, remote_addr: 127.0.0.1:43274, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.146 INFO eid:49 Found repair files: ["031", "031.db"]
Sep 22 23:22:20.147 TRCE incoming request, uri: /newextent/49/data, method: GET, req_id: 7d0b649d-37b4-44f0-87b8-6af1cec1a299, remote_addr: 127.0.0.1:43274, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.147 INFO request completed, latency_us: 345, response_code: 200, uri: /newextent/49/data, method: GET, req_id: 7d0b649d-37b4-44f0-87b8-6af1cec1a299, remote_addr: 127.0.0.1:43274, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.152 TRCE incoming request, uri: /newextent/49/db, method: GET, req_id: 4c85e619-2c09-4f99-aeca-6523e1d273d4, remote_addr: 127.0.0.1:43274, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.152 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/49/db, method: GET, req_id: 4c85e619-2c09-4f99-aeca-6523e1d273d4, remote_addr: 127.0.0.1:43274, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.153 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/031.copy" to "/tmp/downstairs-zrMnlo6G/00/000/031.replace"
Sep 22 23:22:20.153 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.154 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/031.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.155 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/031"
Sep 22 23:22:20.155 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/031.db"
Sep 22 23:22:20.155 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.155 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/031.replace" to "/tmp/downstairs-zrMnlo6G/00/000/031.completed"
Sep 22 23:22:20.155 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.155 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.155 DEBG [0] It's time to notify for 306
Sep 22 23:22:20.155 INFO Completion from [0] id:306 status:true
Sep 22 23:22:20.155 INFO [307/752] Repair commands completed
Sep 22 23:22:20.155 INFO Pop front: ReconcileIO { id: ReconciliationId(307), op: ExtentReopen { repair_id: ReconciliationId(307), extent_id: 49 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.155 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.155 INFO [0] received reconcile message
Sep 22 23:22:20.155 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(307), op: ExtentReopen { repair_id: ReconciliationId(307), extent_id: 49 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.155 INFO [0] client ExtentReopen { repair_id: ReconciliationId(307), extent_id: 49 }
Sep 22 23:22:20.155 INFO [1] received reconcile message
Sep 22 23:22:20.155 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(307), op: ExtentReopen { repair_id: ReconciliationId(307), extent_id: 49 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.155 INFO [1] client ExtentReopen { repair_id: ReconciliationId(307), extent_id: 49 }
Sep 22 23:22:20.155 INFO [2] received reconcile message
Sep 22 23:22:20.155 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(307), op: ExtentReopen { repair_id: ReconciliationId(307), extent_id: 49 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.155 INFO [2] client ExtentReopen { repair_id: ReconciliationId(307), extent_id: 49 }
Sep 22 23:22:20.156 DEBG 307 Reopen extent 49
Sep 22 23:22:20.156 DEBG 307 Reopen extent 49
Sep 22 23:22:20.157 DEBG 307 Reopen extent 49
Sep 22 23:22:20.157 DEBG [2] It's time to notify for 307
Sep 22 23:22:20.157 INFO Completion from [2] id:307 status:true
Sep 22 23:22:20.157 INFO [308/752] Repair commands completed
Sep 22 23:22:20.157 INFO Pop front: ReconcileIO { id: ReconciliationId(308), op: ExtentFlush { repair_id: ReconciliationId(308), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.157 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.157 INFO [0] received reconcile message
Sep 22 23:22:20.157 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(308), op: ExtentFlush { repair_id: ReconciliationId(308), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.157 INFO [0] client ExtentFlush { repair_id: ReconciliationId(308), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.158 INFO [1] received reconcile message
Sep 22 23:22:20.158 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(308), op: ExtentFlush { repair_id: ReconciliationId(308), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.158 INFO [1] client ExtentFlush { repair_id: ReconciliationId(308), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.158 INFO [2] received reconcile message
Sep 22 23:22:20.158 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(308), op: ExtentFlush { repair_id: ReconciliationId(308), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.158 INFO [2] client ExtentFlush { repair_id: ReconciliationId(308), extent_id: 15, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.158 DEBG 308 Flush extent 15 with f:2 g:2
Sep 22 23:22:20.158 DEBG Flush just extent 15 with f:2 and g:2
Sep 22 23:22:20.158 DEBG [1] It's time to notify for 308
Sep 22 23:22:20.158 INFO Completion from [1] id:308 status:true
Sep 22 23:22:20.158 INFO [309/752] Repair commands completed
Sep 22 23:22:20.158 INFO Pop front: ReconcileIO { id: ReconciliationId(309), op: ExtentClose { repair_id: ReconciliationId(309), extent_id: 15 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.158 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.158 INFO [0] received reconcile message
Sep 22 23:22:20.158 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(309), op: ExtentClose { repair_id: ReconciliationId(309), extent_id: 15 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.158 INFO [0] client ExtentClose { repair_id: ReconciliationId(309), extent_id: 15 }
Sep 22 23:22:20.158 INFO [1] received reconcile message
Sep 22 23:22:20.158 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(309), op: ExtentClose { repair_id: ReconciliationId(309), extent_id: 15 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.158 INFO [1] client ExtentClose { repair_id: ReconciliationId(309), extent_id: 15 }
Sep 22 23:22:20.158 INFO [2] received reconcile message
Sep 22 23:22:20.158 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(309), op: ExtentClose { repair_id: ReconciliationId(309), extent_id: 15 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.158 INFO [2] client ExtentClose { repair_id: ReconciliationId(309), extent_id: 15 }
Sep 22 23:22:20.158 DEBG 309 Close extent 15
Sep 22 23:22:20.159 DEBG 309 Close extent 15
Sep 22 23:22:20.159 DEBG 309 Close extent 15
Sep 22 23:22:20.159 DEBG [2] It's time to notify for 309
Sep 22 23:22:20.159 INFO Completion from [2] id:309 status:true
Sep 22 23:22:20.159 INFO [310/752] Repair commands completed
Sep 22 23:22:20.159 INFO Pop front: ReconcileIO { id: ReconciliationId(310), op: ExtentRepair { repair_id: ReconciliationId(310), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.159 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.159 INFO [0] received reconcile message
Sep 22 23:22:20.159 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(310), op: ExtentRepair { repair_id: ReconciliationId(310), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.159 INFO [0] client ExtentRepair { repair_id: ReconciliationId(310), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.159 INFO [0] Sending repair request ReconciliationId(310)
Sep 22 23:22:20.160 INFO [1] received reconcile message
Sep 22 23:22:20.160 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(310), op: ExtentRepair { repair_id: ReconciliationId(310), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.160 INFO [1] client ExtentRepair { repair_id: ReconciliationId(310), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.160 INFO [1] No action required ReconciliationId(310)
Sep 22 23:22:20.160 INFO [2] received reconcile message
Sep 22 23:22:20.160 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(310), op: ExtentRepair { repair_id: ReconciliationId(310), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.160 INFO [2] client ExtentRepair { repair_id: ReconciliationId(310), extent_id: 15, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.160 INFO [2] No action required ReconciliationId(310)
Sep 22 23:22:20.160 DEBG 310 Repair extent 15 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.160 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/00F.copy"
Sep 22 23:22:20.225 INFO accepted connection, remote_addr: 127.0.0.1:60684, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.225 TRCE incoming request, uri: /extent/15/files, method: GET, req_id: 801a47ed-e4f0-481a-8572-f0c327ff4d0d, remote_addr: 127.0.0.1:60684, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.225 INFO request completed, latency_us: 215, response_code: 200, uri: /extent/15/files, method: GET, req_id: 801a47ed-e4f0-481a-8572-f0c327ff4d0d, remote_addr: 127.0.0.1:60684, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.226 INFO eid:15 Found repair files: ["00F", "00F.db"]
Sep 22 23:22:20.226 TRCE incoming request, uri: /newextent/15/data, method: GET, req_id: 76fbd9bd-49a8-4894-8c72-f1f95c2417b7, remote_addr: 127.0.0.1:60684, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.226 INFO request completed, latency_us: 318, response_code: 200, uri: /newextent/15/data, method: GET, req_id: 76fbd9bd-49a8-4894-8c72-f1f95c2417b7, remote_addr: 127.0.0.1:60684, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.231 TRCE incoming request, uri: /newextent/15/db, method: GET, req_id: 8ac1cadf-680f-4a97-bac5-008e2de460aa, remote_addr: 127.0.0.1:60684, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.232 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/15/db, method: GET, req_id: 8ac1cadf-680f-4a97-bac5-008e2de460aa, remote_addr: 127.0.0.1:60684, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.233 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/00F.copy" to "/tmp/downstairs-zrMnlo6G/00/000/00F.replace"
Sep 22 23:22:20.233 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.233 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/00F.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.234 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00F"
Sep 22 23:22:20.234 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00F.db"
Sep 22 23:22:20.234 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.234 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/00F.replace" to "/tmp/downstairs-zrMnlo6G/00/000/00F.completed"
Sep 22 23:22:20.234 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.234 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.234 DEBG [0] It's time to notify for 310
Sep 22 23:22:20.234 INFO Completion from [0] id:310 status:true
Sep 22 23:22:20.234 INFO [311/752] Repair commands completed
Sep 22 23:22:20.234 INFO Pop front: ReconcileIO { id: ReconciliationId(311), op: ExtentReopen { repair_id: ReconciliationId(311), extent_id: 15 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.234 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.234 INFO [0] received reconcile message
Sep 22 23:22:20.234 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(311), op: ExtentReopen { repair_id: ReconciliationId(311), extent_id: 15 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.234 INFO [0] client ExtentReopen { repair_id: ReconciliationId(311), extent_id: 15 }
Sep 22 23:22:20.234 INFO [1] received reconcile message
Sep 22 23:22:20.234 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(311), op: ExtentReopen { repair_id: ReconciliationId(311), extent_id: 15 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.234 INFO [1] client ExtentReopen { repair_id: ReconciliationId(311), extent_id: 15 }
Sep 22 23:22:20.234 INFO [2] received reconcile message
Sep 22 23:22:20.234 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(311), op: ExtentReopen { repair_id: ReconciliationId(311), extent_id: 15 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.234 INFO [2] client ExtentReopen { repair_id: ReconciliationId(311), extent_id: 15 }
Sep 22 23:22:20.235 DEBG 311 Reopen extent 15
Sep 22 23:22:20.235 DEBG 311 Reopen extent 15
Sep 22 23:22:20.236 DEBG 311 Reopen extent 15
Sep 22 23:22:20.236 DEBG [2] It's time to notify for 311
Sep 22 23:22:20.236 INFO Completion from [2] id:311 status:true
Sep 22 23:22:20.236 INFO [312/752] Repair commands completed
Sep 22 23:22:20.236 INFO Pop front: ReconcileIO { id: ReconciliationId(312), op: ExtentFlush { repair_id: ReconciliationId(312), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.236 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.236 INFO [0] received reconcile message
Sep 22 23:22:20.236 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(312), op: ExtentFlush { repair_id: ReconciliationId(312), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.236 INFO [0] client ExtentFlush { repair_id: ReconciliationId(312), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.236 INFO [1] received reconcile message
Sep 22 23:22:20.237 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(312), op: ExtentFlush { repair_id: ReconciliationId(312), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.237 INFO [1] client ExtentFlush { repair_id: ReconciliationId(312), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.237 INFO [2] received reconcile message
Sep 22 23:22:20.237 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(312), op: ExtentFlush { repair_id: ReconciliationId(312), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.237 INFO [2] client ExtentFlush { repair_id: ReconciliationId(312), extent_id: 43, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.237 DEBG 312 Flush extent 43 with f:2 g:2
Sep 22 23:22:20.237 DEBG Flush just extent 43 with f:2 and g:2
Sep 22 23:22:20.237 DEBG [1] It's time to notify for 312
Sep 22 23:22:20.237 INFO Completion from [1] id:312 status:true
Sep 22 23:22:20.237 INFO [313/752] Repair commands completed
Sep 22 23:22:20.237 INFO Pop front: ReconcileIO { id: ReconciliationId(313), op: ExtentClose { repair_id: ReconciliationId(313), extent_id: 43 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.237 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.237 INFO [0] received reconcile message
Sep 22 23:22:20.237 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(313), op: ExtentClose { repair_id: ReconciliationId(313), extent_id: 43 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.237 INFO [0] client ExtentClose { repair_id: ReconciliationId(313), extent_id: 43 }
Sep 22 23:22:20.237 INFO [1] received reconcile message
Sep 22 23:22:20.237 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(313), op: ExtentClose { repair_id: ReconciliationId(313), extent_id: 43 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.237 INFO [1] client ExtentClose { repair_id: ReconciliationId(313), extent_id: 43 }
Sep 22 23:22:20.237 INFO [2] received reconcile message
Sep 22 23:22:20.237 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(313), op: ExtentClose { repair_id: ReconciliationId(313), extent_id: 43 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.237 INFO [2] client ExtentClose { repair_id: ReconciliationId(313), extent_id: 43 }
Sep 22 23:22:20.237 DEBG 313 Close extent 43
Sep 22 23:22:20.238 DEBG 313 Close extent 43
Sep 22 23:22:20.238 DEBG 313 Close extent 43
Sep 22 23:22:20.238 DEBG [2] It's time to notify for 313
Sep 22 23:22:20.238 INFO Completion from [2] id:313 status:true
Sep 22 23:22:20.238 INFO [314/752] Repair commands completed
Sep 22 23:22:20.238 INFO Pop front: ReconcileIO { id: ReconciliationId(314), op: ExtentRepair { repair_id: ReconciliationId(314), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.238 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.238 INFO [0] received reconcile message
Sep 22 23:22:20.238 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(314), op: ExtentRepair { repair_id: ReconciliationId(314), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.238 INFO [0] client ExtentRepair { repair_id: ReconciliationId(314), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.238 INFO [0] Sending repair request ReconciliationId(314)
Sep 22 23:22:20.238 INFO [1] received reconcile message
Sep 22 23:22:20.239 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(314), op: ExtentRepair { repair_id: ReconciliationId(314), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.239 INFO [1] client ExtentRepair { repair_id: ReconciliationId(314), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.239 INFO [1] No action required ReconciliationId(314)
Sep 22 23:22:20.239 INFO [2] received reconcile message
Sep 22 23:22:20.239 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(314), op: ExtentRepair { repair_id: ReconciliationId(314), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.239 INFO [2] client ExtentRepair { repair_id: ReconciliationId(314), extent_id: 43, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.239 INFO [2] No action required ReconciliationId(314)
Sep 22 23:22:20.239 DEBG 314 Repair extent 43 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.239 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/02B.copy"
Sep 22 23:22:20.302 INFO accepted connection, remote_addr: 127.0.0.1:42156, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.302 TRCE incoming request, uri: /extent/43/files, method: GET, req_id: 3b91bffe-4b28-48d9-abe6-9c8f5c4032d8, remote_addr: 127.0.0.1:42156, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.302 INFO request completed, latency_us: 193, response_code: 200, uri: /extent/43/files, method: GET, req_id: 3b91bffe-4b28-48d9-abe6-9c8f5c4032d8, remote_addr: 127.0.0.1:42156, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.302 INFO eid:43 Found repair files: ["02B", "02B.db"]
Sep 22 23:22:20.302 TRCE incoming request, uri: /newextent/43/data, method: GET, req_id: d9f00ab3-0a1d-4684-954e-f87930771d5d, remote_addr: 127.0.0.1:42156, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.303 INFO request completed, latency_us: 291, response_code: 200, uri: /newextent/43/data, method: GET, req_id: d9f00ab3-0a1d-4684-954e-f87930771d5d, remote_addr: 127.0.0.1:42156, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.308 TRCE incoming request, uri: /newextent/43/db, method: GET, req_id: 61d2aecb-9186-4e54-bc26-461522dee8de, remote_addr: 127.0.0.1:42156, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.308 INFO request completed, latency_us: 288, response_code: 200, uri: /newextent/43/db, method: GET, req_id: 61d2aecb-9186-4e54-bc26-461522dee8de, remote_addr: 127.0.0.1:42156, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.309 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/02B.copy" to "/tmp/downstairs-zrMnlo6G/00/000/02B.replace"
Sep 22 23:22:20.309 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.310 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/02B.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.310 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02B"
Sep 22 23:22:20.310 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02B.db"
Sep 22 23:22:20.310 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.310 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/02B.replace" to "/tmp/downstairs-zrMnlo6G/00/000/02B.completed"
Sep 22 23:22:20.310 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.310 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.311 DEBG [0] It's time to notify for 314
Sep 22 23:22:20.311 INFO Completion from [0] id:314 status:true
Sep 22 23:22:20.311 INFO [315/752] Repair commands completed
Sep 22 23:22:20.311 INFO Pop front: ReconcileIO { id: ReconciliationId(315), op: ExtentReopen { repair_id: ReconciliationId(315), extent_id: 43 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.311 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.311 INFO [0] received reconcile message
Sep 22 23:22:20.311 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(315), op: ExtentReopen { repair_id: ReconciliationId(315), extent_id: 43 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.311 INFO [0] client ExtentReopen { repair_id: ReconciliationId(315), extent_id: 43 }
Sep 22 23:22:20.311 INFO [1] received reconcile message
Sep 22 23:22:20.311 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(315), op: ExtentReopen { repair_id: ReconciliationId(315), extent_id: 43 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.311 INFO [1] client ExtentReopen { repair_id: ReconciliationId(315), extent_id: 43 }
Sep 22 23:22:20.311 INFO [2] received reconcile message
Sep 22 23:22:20.311 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(315), op: ExtentReopen { repair_id: ReconciliationId(315), extent_id: 43 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.311 INFO [2] client ExtentReopen { repair_id: ReconciliationId(315), extent_id: 43 }
Sep 22 23:22:20.311 DEBG 315 Reopen extent 43
Sep 22 23:22:20.312 DEBG 315 Reopen extent 43
Sep 22 23:22:20.312 DEBG 315 Reopen extent 43
Sep 22 23:22:20.313 DEBG [2] It's time to notify for 315
Sep 22 23:22:20.313 INFO Completion from [2] id:315 status:true
Sep 22 23:22:20.313 INFO [316/752] Repair commands completed
Sep 22 23:22:20.313 INFO Pop front: ReconcileIO { id: ReconciliationId(316), op: ExtentFlush { repair_id: ReconciliationId(316), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.313 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.313 INFO [0] received reconcile message
Sep 22 23:22:20.313 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(316), op: ExtentFlush { repair_id: ReconciliationId(316), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.313 INFO [0] client ExtentFlush { repair_id: ReconciliationId(316), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.313 INFO [1] received reconcile message
Sep 22 23:22:20.313 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(316), op: ExtentFlush { repair_id: ReconciliationId(316), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.313 INFO [1] client ExtentFlush { repair_id: ReconciliationId(316), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.313 INFO [2] received reconcile message
Sep 22 23:22:20.313 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(316), op: ExtentFlush { repair_id: ReconciliationId(316), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.313 INFO [2] client ExtentFlush { repair_id: ReconciliationId(316), extent_id: 60, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.313 DEBG 316 Flush extent 60 with f:2 g:2
Sep 22 23:22:20.313 DEBG Flush just extent 60 with f:2 and g:2
Sep 22 23:22:20.314 DEBG [1] It's time to notify for 316
Sep 22 23:22:20.314 INFO Completion from [1] id:316 status:true
Sep 22 23:22:20.314 INFO [317/752] Repair commands completed
Sep 22 23:22:20.314 INFO Pop front: ReconcileIO { id: ReconciliationId(317), op: ExtentClose { repair_id: ReconciliationId(317), extent_id: 60 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.314 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.314 INFO [0] received reconcile message
Sep 22 23:22:20.314 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(317), op: ExtentClose { repair_id: ReconciliationId(317), extent_id: 60 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.314 INFO [0] client ExtentClose { repair_id: ReconciliationId(317), extent_id: 60 }
Sep 22 23:22:20.314 INFO [1] received reconcile message
Sep 22 23:22:20.314 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(317), op: ExtentClose { repair_id: ReconciliationId(317), extent_id: 60 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.314 INFO [1] client ExtentClose { repair_id: ReconciliationId(317), extent_id: 60 }
Sep 22 23:22:20.314 INFO [2] received reconcile message
Sep 22 23:22:20.314 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(317), op: ExtentClose { repair_id: ReconciliationId(317), extent_id: 60 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.314 INFO [2] client ExtentClose { repair_id: ReconciliationId(317), extent_id: 60 }
Sep 22 23:22:20.314 DEBG 317 Close extent 60
Sep 22 23:22:20.314 DEBG 317 Close extent 60
Sep 22 23:22:20.315 DEBG 317 Close extent 60
Sep 22 23:22:20.315 DEBG [2] It's time to notify for 317
Sep 22 23:22:20.315 INFO Completion from [2] id:317 status:true
Sep 22 23:22:20.315 INFO [318/752] Repair commands completed
Sep 22 23:22:20.315 INFO Pop front: ReconcileIO { id: ReconciliationId(318), op: ExtentRepair { repair_id: ReconciliationId(318), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.315 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.315 INFO [0] received reconcile message
Sep 22 23:22:20.315 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(318), op: ExtentRepair { repair_id: ReconciliationId(318), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.315 INFO [0] client ExtentRepair { repair_id: ReconciliationId(318), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.315 INFO [0] Sending repair request ReconciliationId(318)
Sep 22 23:22:20.315 INFO [1] received reconcile message
Sep 22 23:22:20.315 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(318), op: ExtentRepair { repair_id: ReconciliationId(318), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.315 INFO [1] client ExtentRepair { repair_id: ReconciliationId(318), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.315 INFO [1] No action required ReconciliationId(318)
Sep 22 23:22:20.315 INFO [2] received reconcile message
Sep 22 23:22:20.315 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(318), op: ExtentRepair { repair_id: ReconciliationId(318), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.315 INFO [2] client ExtentRepair { repair_id: ReconciliationId(318), extent_id: 60, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.315 INFO [2] No action required ReconciliationId(318)
Sep 22 23:22:20.316 DEBG 318 Repair extent 60 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.316 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/03C.copy"
Sep 22 23:22:20.373 DEBG [1] Read AckReady 1081, : downstairs
Sep 22 23:22:20.374 ERRO [1] job id 1082 saw error GenericError("test error")
Sep 22 23:22:20.374 DEBG up_ds_listen was notified
Sep 22 23:22:20.374 DEBG up_ds_listen process 1081
Sep 22 23:22:20.374 DEBG [A] ack job 1081:82, : downstairs
Sep 22 23:22:20.380 INFO accepted connection, remote_addr: 127.0.0.1:34218, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.380 TRCE incoming request, uri: /extent/60/files, method: GET, req_id: 99aaab7a-9b9a-45eb-903b-530a660f81d4, remote_addr: 127.0.0.1:34218, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.380 INFO request completed, latency_us: 201, response_code: 200, uri: /extent/60/files, method: GET, req_id: 99aaab7a-9b9a-45eb-903b-530a660f81d4, remote_addr: 127.0.0.1:34218, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.381 INFO eid:60 Found repair files: ["03C", "03C.db"]
Sep 22 23:22:20.381 TRCE incoming request, uri: /newextent/60/data, method: GET, req_id: 9c1fd412-a54e-47aa-8406-8564f97261eb, remote_addr: 127.0.0.1:34218, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.381 INFO request completed, latency_us: 256, response_code: 200, uri: /newextent/60/data, method: GET, req_id: 9c1fd412-a54e-47aa-8406-8564f97261eb, remote_addr: 127.0.0.1:34218, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.386 TRCE incoming request, uri: /newextent/60/db, method: GET, req_id: b3702ba4-2649-4f51-9f6c-891ac0714979, remote_addr: 127.0.0.1:34218, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.386 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/60/db, method: GET, req_id: b3702ba4-2649-4f51-9f6c-891ac0714979, remote_addr: 127.0.0.1:34218, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.387 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/03C.copy" to "/tmp/downstairs-zrMnlo6G/00/000/03C.replace"
Sep 22 23:22:20.388 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.388 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/03C.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.389 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03C"
Sep 22 23:22:20.389 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03C.db"
Sep 22 23:22:20.389 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.389 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/03C.replace" to "/tmp/downstairs-zrMnlo6G/00/000/03C.completed"
Sep 22 23:22:20.389 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.389 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.389 DEBG [0] It's time to notify for 318
Sep 22 23:22:20.389 INFO Completion from [0] id:318 status:true
Sep 22 23:22:20.389 INFO [319/752] Repair commands completed
Sep 22 23:22:20.389 INFO Pop front: ReconcileIO { id: ReconciliationId(319), op: ExtentReopen { repair_id: ReconciliationId(319), extent_id: 60 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.389 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.389 INFO [0] received reconcile message
Sep 22 23:22:20.389 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(319), op: ExtentReopen { repair_id: ReconciliationId(319), extent_id: 60 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.389 INFO [0] client ExtentReopen { repair_id: ReconciliationId(319), extent_id: 60 }
Sep 22 23:22:20.389 INFO [1] received reconcile message
Sep 22 23:22:20.389 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(319), op: ExtentReopen { repair_id: ReconciliationId(319), extent_id: 60 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.389 INFO [1] client ExtentReopen { repair_id: ReconciliationId(319), extent_id: 60 }
Sep 22 23:22:20.389 INFO [2] received reconcile message
Sep 22 23:22:20.389 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(319), op: ExtentReopen { repair_id: ReconciliationId(319), extent_id: 60 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.389 INFO [2] client ExtentReopen { repair_id: ReconciliationId(319), extent_id: 60 }
Sep 22 23:22:20.390 DEBG 319 Reopen extent 60
Sep 22 23:22:20.390 DEBG 319 Reopen extent 60
Sep 22 23:22:20.391 DEBG 319 Reopen extent 60
Sep 22 23:22:20.391 DEBG [2] It's time to notify for 319
Sep 22 23:22:20.391 INFO Completion from [2] id:319 status:true
Sep 22 23:22:20.391 INFO [320/752] Repair commands completed
Sep 22 23:22:20.391 INFO Pop front: ReconcileIO { id: ReconciliationId(320), op: ExtentFlush { repair_id: ReconciliationId(320), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.391 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.391 INFO [0] received reconcile message
Sep 22 23:22:20.391 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(320), op: ExtentFlush { repair_id: ReconciliationId(320), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.391 INFO [0] client ExtentFlush { repair_id: ReconciliationId(320), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.391 INFO [1] received reconcile message
Sep 22 23:22:20.391 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(320), op: ExtentFlush { repair_id: ReconciliationId(320), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.391 INFO [1] client ExtentFlush { repair_id: ReconciliationId(320), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.391 INFO [2] received reconcile message
Sep 22 23:22:20.391 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(320), op: ExtentFlush { repair_id: ReconciliationId(320), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.392 INFO [2] client ExtentFlush { repair_id: ReconciliationId(320), extent_id: 18, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.392 DEBG 320 Flush extent 18 with f:2 g:2
Sep 22 23:22:20.392 DEBG Flush just extent 18 with f:2 and g:2
Sep 22 23:22:20.392 DEBG [1] It's time to notify for 320
Sep 22 23:22:20.392 INFO Completion from [1] id:320 status:true
Sep 22 23:22:20.392 INFO [321/752] Repair commands completed
Sep 22 23:22:20.392 INFO Pop front: ReconcileIO { id: ReconciliationId(321), op: ExtentClose { repair_id: ReconciliationId(321), extent_id: 18 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.392 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.392 INFO [0] received reconcile message
Sep 22 23:22:20.392 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(321), op: ExtentClose { repair_id: ReconciliationId(321), extent_id: 18 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.392 INFO [0] client ExtentClose { repair_id: ReconciliationId(321), extent_id: 18 }
Sep 22 23:22:20.392 INFO [1] received reconcile message
Sep 22 23:22:20.392 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(321), op: ExtentClose { repair_id: ReconciliationId(321), extent_id: 18 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.392 INFO [1] client ExtentClose { repair_id: ReconciliationId(321), extent_id: 18 }
Sep 22 23:22:20.392 INFO [2] received reconcile message
Sep 22 23:22:20.392 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(321), op: ExtentClose { repair_id: ReconciliationId(321), extent_id: 18 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.392 INFO [2] client ExtentClose { repair_id: ReconciliationId(321), extent_id: 18 }
Sep 22 23:22:20.392 DEBG 321 Close extent 18
Sep 22 23:22:20.392 DEBG 321 Close extent 18
Sep 22 23:22:20.393 DEBG 321 Close extent 18
Sep 22 23:22:20.393 DEBG [2] It's time to notify for 321
Sep 22 23:22:20.393 INFO Completion from [2] id:321 status:true
Sep 22 23:22:20.393 INFO [322/752] Repair commands completed
Sep 22 23:22:20.393 INFO Pop front: ReconcileIO { id: ReconciliationId(322), op: ExtentRepair { repair_id: ReconciliationId(322), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.393 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.393 INFO [0] received reconcile message
Sep 22 23:22:20.393 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(322), op: ExtentRepair { repair_id: ReconciliationId(322), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.393 INFO [0] client ExtentRepair { repair_id: ReconciliationId(322), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.393 INFO [0] Sending repair request ReconciliationId(322)
Sep 22 23:22:20.393 INFO [1] received reconcile message
Sep 22 23:22:20.393 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(322), op: ExtentRepair { repair_id: ReconciliationId(322), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.393 INFO [1] client ExtentRepair { repair_id: ReconciliationId(322), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.394 INFO [1] No action required ReconciliationId(322)
Sep 22 23:22:20.394 INFO [2] received reconcile message
Sep 22 23:22:20.394 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(322), op: ExtentRepair { repair_id: ReconciliationId(322), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.394 INFO [2] client ExtentRepair { repair_id: ReconciliationId(322), extent_id: 18, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.394 INFO [2] No action required ReconciliationId(322)
Sep 22 23:22:20.394 DEBG 322 Repair extent 18 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.394 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/012.copy"
Sep 22 23:22:20.427 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:20.430 DEBG Flush     :1082 extent_limit None deps:[JobId(1081), JobId(1080)] res:true f:30 g:1
Sep 22 23:22:20.430 INFO [lossy] sleeping 1 second
Sep 22 23:22:20.430 INFO [lossy] skipping 1081
Sep 22 23:22:20.430 INFO [lossy] skipping 1082
Sep 22 23:22:20.430 WARN returning error on read!
Sep 22 23:22:20.430 DEBG Read      :1081 deps:[JobId(1080)] res:false
Sep 22 23:22:20.430 INFO [lossy] skipping 1082
Sep 22 23:22:20.430 INFO [lossy] skipping 1081
Sep 22 23:22:20.430 INFO [lossy] skipping 1082
Sep 22 23:22:20.430 INFO [lossy] skipping 1081
Sep 22 23:22:20.436 DEBG Read      :1081 deps:[JobId(1080)] res:true
Sep 22 23:22:20.459 INFO accepted connection, remote_addr: 127.0.0.1:44148, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.459 TRCE incoming request, uri: /extent/18/files, method: GET, req_id: 190edafc-ed12-473d-9d18-a7652c2a94ea, remote_addr: 127.0.0.1:44148, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.459 INFO request completed, latency_us: 242, response_code: 200, uri: /extent/18/files, method: GET, req_id: 190edafc-ed12-473d-9d18-a7652c2a94ea, remote_addr: 127.0.0.1:44148, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.460 INFO eid:18 Found repair files: ["012", "012.db"]
Sep 22 23:22:20.460 TRCE incoming request, uri: /newextent/18/data, method: GET, req_id: e9cb2297-7515-444d-b235-1d25c804d296, remote_addr: 127.0.0.1:44148, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.460 INFO request completed, latency_us: 349, response_code: 200, uri: /newextent/18/data, method: GET, req_id: e9cb2297-7515-444d-b235-1d25c804d296, remote_addr: 127.0.0.1:44148, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.465 TRCE incoming request, uri: /newextent/18/db, method: GET, req_id: d0505bf2-8c91-4f19-86e9-af7be9578d9c, remote_addr: 127.0.0.1:44148, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.466 INFO request completed, latency_us: 352, response_code: 200, uri: /newextent/18/db, method: GET, req_id: d0505bf2-8c91-4f19-86e9-af7be9578d9c, remote_addr: 127.0.0.1:44148, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.467 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/012.copy" to "/tmp/downstairs-zrMnlo6G/00/000/012.replace"
Sep 22 23:22:20.467 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.468 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/012.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.468 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/012"
Sep 22 23:22:20.468 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/012.db"
Sep 22 23:22:20.468 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.468 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/012.replace" to "/tmp/downstairs-zrMnlo6G/00/000/012.completed"
Sep 22 23:22:20.468 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.468 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.469 DEBG [0] It's time to notify for 322
Sep 22 23:22:20.469 INFO Completion from [0] id:322 status:true
Sep 22 23:22:20.469 INFO [323/752] Repair commands completed
Sep 22 23:22:20.469 INFO Pop front: ReconcileIO { id: ReconciliationId(323), op: ExtentReopen { repair_id: ReconciliationId(323), extent_id: 18 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.469 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.469 INFO [0] received reconcile message
Sep 22 23:22:20.469 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(323), op: ExtentReopen { repair_id: ReconciliationId(323), extent_id: 18 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.469 INFO [0] client ExtentReopen { repair_id: ReconciliationId(323), extent_id: 18 }
Sep 22 23:22:20.469 INFO [1] received reconcile message
Sep 22 23:22:20.469 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(323), op: ExtentReopen { repair_id: ReconciliationId(323), extent_id: 18 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.469 INFO [1] client ExtentReopen { repair_id: ReconciliationId(323), extent_id: 18 }
Sep 22 23:22:20.469 INFO [2] received reconcile message
Sep 22 23:22:20.469 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(323), op: ExtentReopen { repair_id: ReconciliationId(323), extent_id: 18 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.469 INFO [2] client ExtentReopen { repair_id: ReconciliationId(323), extent_id: 18 }
Sep 22 23:22:20.469 DEBG 323 Reopen extent 18
Sep 22 23:22:20.470 DEBG 323 Reopen extent 18
Sep 22 23:22:20.470 DEBG 323 Reopen extent 18
Sep 22 23:22:20.471 DEBG [2] It's time to notify for 323
Sep 22 23:22:20.471 INFO Completion from [2] id:323 status:true
Sep 22 23:22:20.471 INFO [324/752] Repair commands completed
Sep 22 23:22:20.471 INFO Pop front: ReconcileIO { id: ReconciliationId(324), op: ExtentFlush { repair_id: ReconciliationId(324), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.471 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.471 INFO [0] received reconcile message
Sep 22 23:22:20.471 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(324), op: ExtentFlush { repair_id: ReconciliationId(324), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.471 INFO [0] client ExtentFlush { repair_id: ReconciliationId(324), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.471 INFO [1] received reconcile message
Sep 22 23:22:20.471 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(324), op: ExtentFlush { repair_id: ReconciliationId(324), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.471 INFO [1] client ExtentFlush { repair_id: ReconciliationId(324), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.471 INFO [2] received reconcile message
Sep 22 23:22:20.471 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(324), op: ExtentFlush { repair_id: ReconciliationId(324), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.471 INFO [2] client ExtentFlush { repair_id: ReconciliationId(324), extent_id: 110, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.472 DEBG 324 Flush extent 110 with f:2 g:2
Sep 22 23:22:20.472 DEBG Flush just extent 110 with f:2 and g:2
Sep 22 23:22:20.472 DEBG [1] It's time to notify for 324
Sep 22 23:22:20.472 INFO Completion from [1] id:324 status:true
Sep 22 23:22:20.472 INFO [325/752] Repair commands completed
Sep 22 23:22:20.472 INFO Pop front: ReconcileIO { id: ReconciliationId(325), op: ExtentClose { repair_id: ReconciliationId(325), extent_id: 110 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.472 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.472 INFO [0] received reconcile message
Sep 22 23:22:20.472 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(325), op: ExtentClose { repair_id: ReconciliationId(325), extent_id: 110 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.472 INFO [0] client ExtentClose { repair_id: ReconciliationId(325), extent_id: 110 }
Sep 22 23:22:20.472 INFO [1] received reconcile message
Sep 22 23:22:20.472 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(325), op: ExtentClose { repair_id: ReconciliationId(325), extent_id: 110 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.472 INFO [1] client ExtentClose { repair_id: ReconciliationId(325), extent_id: 110 }
Sep 22 23:22:20.472 INFO [2] received reconcile message
Sep 22 23:22:20.472 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(325), op: ExtentClose { repair_id: ReconciliationId(325), extent_id: 110 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.472 INFO [2] client ExtentClose { repair_id: ReconciliationId(325), extent_id: 110 }
Sep 22 23:22:20.472 DEBG 325 Close extent 110
Sep 22 23:22:20.472 DEBG 325 Close extent 110
Sep 22 23:22:20.473 DEBG 325 Close extent 110
Sep 22 23:22:20.473 DEBG [2] It's time to notify for 325
Sep 22 23:22:20.473 INFO Completion from [2] id:325 status:true
Sep 22 23:22:20.473 INFO [326/752] Repair commands completed
Sep 22 23:22:20.473 INFO Pop front: ReconcileIO { id: ReconciliationId(326), op: ExtentRepair { repair_id: ReconciliationId(326), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.473 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.473 INFO [0] received reconcile message
Sep 22 23:22:20.473 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(326), op: ExtentRepair { repair_id: ReconciliationId(326), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.473 INFO [0] client ExtentRepair { repair_id: ReconciliationId(326), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.473 INFO [0] Sending repair request ReconciliationId(326)
Sep 22 23:22:20.473 INFO [1] received reconcile message
Sep 22 23:22:20.473 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(326), op: ExtentRepair { repair_id: ReconciliationId(326), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.473 INFO [1] client ExtentRepair { repair_id: ReconciliationId(326), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.473 INFO [1] No action required ReconciliationId(326)
Sep 22 23:22:20.473 INFO [2] received reconcile message
Sep 22 23:22:20.473 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(326), op: ExtentRepair { repair_id: ReconciliationId(326), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.474 INFO [2] client ExtentRepair { repair_id: ReconciliationId(326), extent_id: 110, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.474 INFO [2] No action required ReconciliationId(326)
Sep 22 23:22:20.474 DEBG 326 Repair extent 110 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.474 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/06E.copy"
Sep 22 23:22:20.506 DEBG IO Read  1083 has deps [JobId(1082)]
Sep 22 23:22:20.538 INFO accepted connection, remote_addr: 127.0.0.1:58718, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.538 TRCE incoming request, uri: /extent/110/files, method: GET, req_id: 4231e696-e5ee-4677-ab8e-0e04ca1cbe20, remote_addr: 127.0.0.1:58718, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.539 INFO request completed, latency_us: 204, response_code: 200, uri: /extent/110/files, method: GET, req_id: 4231e696-e5ee-4677-ab8e-0e04ca1cbe20, remote_addr: 127.0.0.1:58718, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.539 INFO eid:110 Found repair files: ["06E", "06E.db"]
Sep 22 23:22:20.539 TRCE incoming request, uri: /newextent/110/data, method: GET, req_id: fca3749f-3f15-4d66-bc9d-5c29cf8611b7, remote_addr: 127.0.0.1:58718, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.540 INFO request completed, latency_us: 326, response_code: 200, uri: /newextent/110/data, method: GET, req_id: fca3749f-3f15-4d66-bc9d-5c29cf8611b7, remote_addr: 127.0.0.1:58718, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.544 TRCE incoming request, uri: /newextent/110/db, method: GET, req_id: ebd033e8-311b-4eed-a5d1-b9083be6ce8a, remote_addr: 127.0.0.1:58718, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.544 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/110/db, method: GET, req_id: ebd033e8-311b-4eed-a5d1-b9083be6ce8a, remote_addr: 127.0.0.1:58718, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.546 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/06E.copy" to "/tmp/downstairs-zrMnlo6G/00/000/06E.replace"
Sep 22 23:22:20.546 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.546 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/06E.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.547 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06E"
Sep 22 23:22:20.547 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06E.db"
Sep 22 23:22:20.547 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.547 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/06E.replace" to "/tmp/downstairs-zrMnlo6G/00/000/06E.completed"
Sep 22 23:22:20.547 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.547 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.547 DEBG [0] It's time to notify for 326
Sep 22 23:22:20.547 INFO Completion from [0] id:326 status:true
Sep 22 23:22:20.547 INFO [327/752] Repair commands completed
Sep 22 23:22:20.547 INFO Pop front: ReconcileIO { id: ReconciliationId(327), op: ExtentReopen { repair_id: ReconciliationId(327), extent_id: 110 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.547 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.547 INFO [0] received reconcile message
Sep 22 23:22:20.547 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(327), op: ExtentReopen { repair_id: ReconciliationId(327), extent_id: 110 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.547 INFO [0] client ExtentReopen { repair_id: ReconciliationId(327), extent_id: 110 }
Sep 22 23:22:20.547 INFO [1] received reconcile message
Sep 22 23:22:20.547 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(327), op: ExtentReopen { repair_id: ReconciliationId(327), extent_id: 110 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.548 INFO [1] client ExtentReopen { repair_id: ReconciliationId(327), extent_id: 110 }
Sep 22 23:22:20.548 INFO [2] received reconcile message
Sep 22 23:22:20.548 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(327), op: ExtentReopen { repair_id: ReconciliationId(327), extent_id: 110 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.548 INFO [2] client ExtentReopen { repair_id: ReconciliationId(327), extent_id: 110 }
Sep 22 23:22:20.548 DEBG 327 Reopen extent 110
Sep 22 23:22:20.548 DEBG 327 Reopen extent 110
Sep 22 23:22:20.549 DEBG 327 Reopen extent 110
Sep 22 23:22:20.549 DEBG [2] It's time to notify for 327
Sep 22 23:22:20.549 INFO Completion from [2] id:327 status:true
Sep 22 23:22:20.549 INFO [328/752] Repair commands completed
Sep 22 23:22:20.549 INFO Pop front: ReconcileIO { id: ReconciliationId(328), op: ExtentFlush { repair_id: ReconciliationId(328), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.550 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.550 INFO [0] received reconcile message
Sep 22 23:22:20.550 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(328), op: ExtentFlush { repair_id: ReconciliationId(328), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.550 INFO [0] client ExtentFlush { repair_id: ReconciliationId(328), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.550 INFO [1] received reconcile message
Sep 22 23:22:20.550 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(328), op: ExtentFlush { repair_id: ReconciliationId(328), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.550 INFO [1] client ExtentFlush { repair_id: ReconciliationId(328), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.550 INFO [2] received reconcile message
Sep 22 23:22:20.550 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(328), op: ExtentFlush { repair_id: ReconciliationId(328), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.550 INFO [2] client ExtentFlush { repair_id: ReconciliationId(328), extent_id: 136, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.550 DEBG 328 Flush extent 136 with f:2 g:2
Sep 22 23:22:20.550 DEBG Flush just extent 136 with f:2 and g:2
Sep 22 23:22:20.550 DEBG [1] It's time to notify for 328
Sep 22 23:22:20.550 INFO Completion from [1] id:328 status:true
Sep 22 23:22:20.550 INFO [329/752] Repair commands completed
Sep 22 23:22:20.550 INFO Pop front: ReconcileIO { id: ReconciliationId(329), op: ExtentClose { repair_id: ReconciliationId(329), extent_id: 136 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.550 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.550 INFO [0] received reconcile message
Sep 22 23:22:20.550 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(329), op: ExtentClose { repair_id: ReconciliationId(329), extent_id: 136 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.550 INFO [0] client ExtentClose { repair_id: ReconciliationId(329), extent_id: 136 }
Sep 22 23:22:20.550 INFO [1] received reconcile message
Sep 22 23:22:20.550 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(329), op: ExtentClose { repair_id: ReconciliationId(329), extent_id: 136 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.550 INFO [1] client ExtentClose { repair_id: ReconciliationId(329), extent_id: 136 }
Sep 22 23:22:20.550 INFO [2] received reconcile message
Sep 22 23:22:20.550 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(329), op: ExtentClose { repair_id: ReconciliationId(329), extent_id: 136 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.550 INFO [2] client ExtentClose { repair_id: ReconciliationId(329), extent_id: 136 }
Sep 22 23:22:20.550 DEBG 329 Close extent 136
Sep 22 23:22:20.551 DEBG 329 Close extent 136
Sep 22 23:22:20.551 DEBG 329 Close extent 136
Sep 22 23:22:20.551 DEBG [2] It's time to notify for 329
Sep 22 23:22:20.551 INFO Completion from [2] id:329 status:true
Sep 22 23:22:20.552 INFO [330/752] Repair commands completed
Sep 22 23:22:20.552 INFO Pop front: ReconcileIO { id: ReconciliationId(330), op: ExtentRepair { repair_id: ReconciliationId(330), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.552 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.552 INFO [0] received reconcile message
Sep 22 23:22:20.552 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(330), op: ExtentRepair { repair_id: ReconciliationId(330), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.552 INFO [0] client ExtentRepair { repair_id: ReconciliationId(330), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.552 INFO [0] Sending repair request ReconciliationId(330)
Sep 22 23:22:20.552 INFO [1] received reconcile message
Sep 22 23:22:20.552 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(330), op: ExtentRepair { repair_id: ReconciliationId(330), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.552 INFO [1] client ExtentRepair { repair_id: ReconciliationId(330), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.552 INFO [1] No action required ReconciliationId(330)
Sep 22 23:22:20.552 INFO [2] received reconcile message
Sep 22 23:22:20.552 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(330), op: ExtentRepair { repair_id: ReconciliationId(330), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.552 INFO [2] client ExtentRepair { repair_id: ReconciliationId(330), extent_id: 136, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.552 INFO [2] No action required ReconciliationId(330)
Sep 22 23:22:20.552 DEBG 330 Repair extent 136 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.552 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/088.copy"
Sep 22 23:22:20.614 INFO accepted connection, remote_addr: 127.0.0.1:63364, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.615 TRCE incoming request, uri: /extent/136/files, method: GET, req_id: fa98f1b4-02b6-46c2-9a16-c22c578c7be0, remote_addr: 127.0.0.1:63364, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.615 INFO request completed, latency_us: 195, response_code: 200, uri: /extent/136/files, method: GET, req_id: fa98f1b4-02b6-46c2-9a16-c22c578c7be0, remote_addr: 127.0.0.1:63364, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.615 INFO eid:136 Found repair files: ["088", "088.db"]
Sep 22 23:22:20.615 TRCE incoming request, uri: /newextent/136/data, method: GET, req_id: f4a3ccf3-9f30-49cb-a421-74a6afa13a96, remote_addr: 127.0.0.1:63364, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.616 INFO request completed, latency_us: 305, response_code: 200, uri: /newextent/136/data, method: GET, req_id: f4a3ccf3-9f30-49cb-a421-74a6afa13a96, remote_addr: 127.0.0.1:63364, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.620 TRCE incoming request, uri: /newextent/136/db, method: GET, req_id: 6fee2191-cd90-421c-82af-0faf89c5e3ce, remote_addr: 127.0.0.1:63364, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.621 INFO request completed, latency_us: 297, response_code: 200, uri: /newextent/136/db, method: GET, req_id: 6fee2191-cd90-421c-82af-0faf89c5e3ce, remote_addr: 127.0.0.1:63364, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.622 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/088.copy" to "/tmp/downstairs-zrMnlo6G/00/000/088.replace"
Sep 22 23:22:20.622 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.623 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/088.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.623 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/088"
Sep 22 23:22:20.623 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/088.db"
Sep 22 23:22:20.623 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.623 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/088.replace" to "/tmp/downstairs-zrMnlo6G/00/000/088.completed"
Sep 22 23:22:20.623 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.623 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.623 DEBG [0] It's time to notify for 330
Sep 22 23:22:20.623 INFO Completion from [0] id:330 status:true
Sep 22 23:22:20.623 INFO [331/752] Repair commands completed
Sep 22 23:22:20.623 INFO Pop front: ReconcileIO { id: ReconciliationId(331), op: ExtentReopen { repair_id: ReconciliationId(331), extent_id: 136 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.623 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.624 INFO [0] received reconcile message
Sep 22 23:22:20.624 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(331), op: ExtentReopen { repair_id: ReconciliationId(331), extent_id: 136 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.624 INFO [0] client ExtentReopen { repair_id: ReconciliationId(331), extent_id: 136 }
Sep 22 23:22:20.624 INFO [1] received reconcile message
Sep 22 23:22:20.624 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(331), op: ExtentReopen { repair_id: ReconciliationId(331), extent_id: 136 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.624 INFO [1] client ExtentReopen { repair_id: ReconciliationId(331), extent_id: 136 }
Sep 22 23:22:20.624 INFO [2] received reconcile message
Sep 22 23:22:20.624 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(331), op: ExtentReopen { repair_id: ReconciliationId(331), extent_id: 136 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.624 INFO [2] client ExtentReopen { repair_id: ReconciliationId(331), extent_id: 136 }
Sep 22 23:22:20.624 DEBG 331 Reopen extent 136
Sep 22 23:22:20.624 DEBG 331 Reopen extent 136
Sep 22 23:22:20.625 DEBG 331 Reopen extent 136
Sep 22 23:22:20.626 DEBG [2] It's time to notify for 331
Sep 22 23:22:20.626 INFO Completion from [2] id:331 status:true
Sep 22 23:22:20.626 INFO [332/752] Repair commands completed
Sep 22 23:22:20.626 INFO Pop front: ReconcileIO { id: ReconciliationId(332), op: ExtentFlush { repair_id: ReconciliationId(332), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.626 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.626 INFO [0] received reconcile message
Sep 22 23:22:20.626 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(332), op: ExtentFlush { repair_id: ReconciliationId(332), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.626 INFO [0] client ExtentFlush { repair_id: ReconciliationId(332), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.626 INFO [1] received reconcile message
Sep 22 23:22:20.626 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(332), op: ExtentFlush { repair_id: ReconciliationId(332), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.626 INFO [1] client ExtentFlush { repair_id: ReconciliationId(332), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.626 INFO [2] received reconcile message
Sep 22 23:22:20.626 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(332), op: ExtentFlush { repair_id: ReconciliationId(332), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.626 INFO [2] client ExtentFlush { repair_id: ReconciliationId(332), extent_id: 61, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.626 DEBG 332 Flush extent 61 with f:2 g:2
Sep 22 23:22:20.626 DEBG Flush just extent 61 with f:2 and g:2
Sep 22 23:22:20.626 DEBG [1] It's time to notify for 332
Sep 22 23:22:20.626 INFO Completion from [1] id:332 status:true
Sep 22 23:22:20.626 INFO [333/752] Repair commands completed
Sep 22 23:22:20.626 INFO Pop front: ReconcileIO { id: ReconciliationId(333), op: ExtentClose { repair_id: ReconciliationId(333), extent_id: 61 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.626 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.626 INFO [0] received reconcile message
Sep 22 23:22:20.626 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(333), op: ExtentClose { repair_id: ReconciliationId(333), extent_id: 61 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.626 INFO [0] client ExtentClose { repair_id: ReconciliationId(333), extent_id: 61 }
Sep 22 23:22:20.626 INFO [1] received reconcile message
Sep 22 23:22:20.626 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(333), op: ExtentClose { repair_id: ReconciliationId(333), extent_id: 61 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.626 INFO [1] client ExtentClose { repair_id: ReconciliationId(333), extent_id: 61 }
Sep 22 23:22:20.626 INFO [2] received reconcile message
Sep 22 23:22:20.626 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(333), op: ExtentClose { repair_id: ReconciliationId(333), extent_id: 61 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.626 INFO [2] client ExtentClose { repair_id: ReconciliationId(333), extent_id: 61 }
Sep 22 23:22:20.627 DEBG 333 Close extent 61
Sep 22 23:22:20.627 DEBG 333 Close extent 61
Sep 22 23:22:20.627 DEBG 333 Close extent 61
Sep 22 23:22:20.628 DEBG [2] It's time to notify for 333
Sep 22 23:22:20.628 INFO Completion from [2] id:333 status:true
Sep 22 23:22:20.628 INFO [334/752] Repair commands completed
Sep 22 23:22:20.628 INFO Pop front: ReconcileIO { id: ReconciliationId(334), op: ExtentRepair { repair_id: ReconciliationId(334), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.628 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.628 INFO [0] received reconcile message
Sep 22 23:22:20.628 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(334), op: ExtentRepair { repair_id: ReconciliationId(334), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.628 INFO [0] client ExtentRepair { repair_id: ReconciliationId(334), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.628 INFO [0] Sending repair request ReconciliationId(334)
Sep 22 23:22:20.628 INFO [1] received reconcile message
Sep 22 23:22:20.628 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(334), op: ExtentRepair { repair_id: ReconciliationId(334), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.628 INFO [1] client ExtentRepair { repair_id: ReconciliationId(334), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.628 INFO [1] No action required ReconciliationId(334)
Sep 22 23:22:20.628 INFO [2] received reconcile message
Sep 22 23:22:20.628 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(334), op: ExtentRepair { repair_id: ReconciliationId(334), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.628 INFO [2] client ExtentRepair { repair_id: ReconciliationId(334), extent_id: 61, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.628 INFO [2] No action required ReconciliationId(334)
Sep 22 23:22:20.628 DEBG 334 Repair extent 61 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.628 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/03D.copy"
Sep 22 23:22:20.694 INFO accepted connection, remote_addr: 127.0.0.1:51628, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.694 TRCE incoming request, uri: /extent/61/files, method: GET, req_id: 2ed5c2a6-37d2-4d0d-a854-29afc6e81f41, remote_addr: 127.0.0.1:51628, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.694 INFO request completed, latency_us: 228, response_code: 200, uri: /extent/61/files, method: GET, req_id: 2ed5c2a6-37d2-4d0d-a854-29afc6e81f41, remote_addr: 127.0.0.1:51628, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.695 INFO eid:61 Found repair files: ["03D", "03D.db"]
Sep 22 23:22:20.695 TRCE incoming request, uri: /newextent/61/data, method: GET, req_id: bb61d23f-805b-46bf-9dce-f4d981c96af7, remote_addr: 127.0.0.1:51628, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.695 INFO request completed, latency_us: 344, response_code: 200, uri: /newextent/61/data, method: GET, req_id: bb61d23f-805b-46bf-9dce-f4d981c96af7, remote_addr: 127.0.0.1:51628, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.700 TRCE incoming request, uri: /newextent/61/db, method: GET, req_id: bac254bc-c714-4bea-beb4-a4b6a3ff9746, remote_addr: 127.0.0.1:51628, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.700 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/61/db, method: GET, req_id: bac254bc-c714-4bea-beb4-a4b6a3ff9746, remote_addr: 127.0.0.1:51628, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.702 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/03D.copy" to "/tmp/downstairs-zrMnlo6G/00/000/03D.replace"
Sep 22 23:22:20.702 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.702 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/03D.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.703 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03D"
Sep 22 23:22:20.703 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03D.db"
Sep 22 23:22:20.703 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.703 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/03D.replace" to "/tmp/downstairs-zrMnlo6G/00/000/03D.completed"
Sep 22 23:22:20.703 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.703 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.703 DEBG [0] It's time to notify for 334
Sep 22 23:22:20.703 INFO Completion from [0] id:334 status:true
Sep 22 23:22:20.703 INFO [335/752] Repair commands completed
Sep 22 23:22:20.703 INFO Pop front: ReconcileIO { id: ReconciliationId(335), op: ExtentReopen { repair_id: ReconciliationId(335), extent_id: 61 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.703 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.703 INFO [0] received reconcile message
Sep 22 23:22:20.703 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(335), op: ExtentReopen { repair_id: ReconciliationId(335), extent_id: 61 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.703 INFO [0] client ExtentReopen { repair_id: ReconciliationId(335), extent_id: 61 }
Sep 22 23:22:20.704 INFO [1] received reconcile message
Sep 22 23:22:20.704 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(335), op: ExtentReopen { repair_id: ReconciliationId(335), extent_id: 61 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.704 INFO [1] client ExtentReopen { repair_id: ReconciliationId(335), extent_id: 61 }
Sep 22 23:22:20.704 INFO [2] received reconcile message
Sep 22 23:22:20.704 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(335), op: ExtentReopen { repair_id: ReconciliationId(335), extent_id: 61 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.704 INFO [2] client ExtentReopen { repair_id: ReconciliationId(335), extent_id: 61 }
Sep 22 23:22:20.704 DEBG 335 Reopen extent 61
Sep 22 23:22:20.704 DEBG 335 Reopen extent 61
Sep 22 23:22:20.705 DEBG 335 Reopen extent 61
Sep 22 23:22:20.706 DEBG [2] It's time to notify for 335
Sep 22 23:22:20.706 INFO Completion from [2] id:335 status:true
Sep 22 23:22:20.706 INFO [336/752] Repair commands completed
Sep 22 23:22:20.706 INFO Pop front: ReconcileIO { id: ReconciliationId(336), op: ExtentFlush { repair_id: ReconciliationId(336), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.706 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.706 INFO [0] received reconcile message
Sep 22 23:22:20.706 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(336), op: ExtentFlush { repair_id: ReconciliationId(336), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.706 INFO [0] client ExtentFlush { repair_id: ReconciliationId(336), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.706 INFO [1] received reconcile message
Sep 22 23:22:20.706 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(336), op: ExtentFlush { repair_id: ReconciliationId(336), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.706 INFO [1] client ExtentFlush { repair_id: ReconciliationId(336), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.706 INFO [2] received reconcile message
Sep 22 23:22:20.706 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(336), op: ExtentFlush { repair_id: ReconciliationId(336), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.706 INFO [2] client ExtentFlush { repair_id: ReconciliationId(336), extent_id: 119, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.706 DEBG 336 Flush extent 119 with f:2 g:2
Sep 22 23:22:20.706 DEBG Flush just extent 119 with f:2 and g:2
Sep 22 23:22:20.706 DEBG [1] It's time to notify for 336
Sep 22 23:22:20.706 INFO Completion from [1] id:336 status:true
Sep 22 23:22:20.706 INFO [337/752] Repair commands completed
Sep 22 23:22:20.706 INFO Pop front: ReconcileIO { id: ReconciliationId(337), op: ExtentClose { repair_id: ReconciliationId(337), extent_id: 119 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.706 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.706 INFO [0] received reconcile message
Sep 22 23:22:20.706 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(337), op: ExtentClose { repair_id: ReconciliationId(337), extent_id: 119 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.706 INFO [0] client ExtentClose { repair_id: ReconciliationId(337), extent_id: 119 }
Sep 22 23:22:20.706 INFO [1] received reconcile message
Sep 22 23:22:20.706 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(337), op: ExtentClose { repair_id: ReconciliationId(337), extent_id: 119 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.706 INFO [1] client ExtentClose { repair_id: ReconciliationId(337), extent_id: 119 }
Sep 22 23:22:20.706 INFO [2] received reconcile message
Sep 22 23:22:20.706 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(337), op: ExtentClose { repair_id: ReconciliationId(337), extent_id: 119 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.707 INFO [2] client ExtentClose { repair_id: ReconciliationId(337), extent_id: 119 }
Sep 22 23:22:20.707 DEBG 337 Close extent 119
Sep 22 23:22:20.707 DEBG 337 Close extent 119
Sep 22 23:22:20.707 DEBG 337 Close extent 119
Sep 22 23:22:20.708 DEBG [2] It's time to notify for 337
Sep 22 23:22:20.708 INFO Completion from [2] id:337 status:true
Sep 22 23:22:20.708 INFO [338/752] Repair commands completed
Sep 22 23:22:20.708 INFO Pop front: ReconcileIO { id: ReconciliationId(338), op: ExtentRepair { repair_id: ReconciliationId(338), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.708 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.708 INFO [0] received reconcile message
Sep 22 23:22:20.708 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(338), op: ExtentRepair { repair_id: ReconciliationId(338), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.708 INFO [0] client ExtentRepair { repair_id: ReconciliationId(338), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.708 INFO [0] Sending repair request ReconciliationId(338)
Sep 22 23:22:20.708 INFO [1] received reconcile message
Sep 22 23:22:20.708 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(338), op: ExtentRepair { repair_id: ReconciliationId(338), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.708 INFO [1] client ExtentRepair { repair_id: ReconciliationId(338), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.708 INFO [1] No action required ReconciliationId(338)
Sep 22 23:22:20.708 INFO [2] received reconcile message
Sep 22 23:22:20.708 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(338), op: ExtentRepair { repair_id: ReconciliationId(338), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.708 INFO [2] client ExtentRepair { repair_id: ReconciliationId(338), extent_id: 119, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.708 INFO [2] No action required ReconciliationId(338)
Sep 22 23:22:20.708 DEBG 338 Repair extent 119 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.708 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/077.copy"
Sep 22 23:22:20.772 INFO accepted connection, remote_addr: 127.0.0.1:57841, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.772 TRCE incoming request, uri: /extent/119/files, method: GET, req_id: a3003b4f-a27b-453f-a3ce-cf6ee617df66, remote_addr: 127.0.0.1:57841, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.772 INFO request completed, latency_us: 208, response_code: 200, uri: /extent/119/files, method: GET, req_id: a3003b4f-a27b-453f-a3ce-cf6ee617df66, remote_addr: 127.0.0.1:57841, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.772 INFO eid:119 Found repair files: ["077", "077.db"]
Sep 22 23:22:20.773 TRCE incoming request, uri: /newextent/119/data, method: GET, req_id: da6f6d79-c613-49ed-83b1-1b1f95301334, remote_addr: 127.0.0.1:57841, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.773 INFO request completed, latency_us: 263, response_code: 200, uri: /newextent/119/data, method: GET, req_id: da6f6d79-c613-49ed-83b1-1b1f95301334, remote_addr: 127.0.0.1:57841, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.778 TRCE incoming request, uri: /newextent/119/db, method: GET, req_id: 70454771-459b-4510-8e83-c2073313f6b7, remote_addr: 127.0.0.1:57841, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.778 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/119/db, method: GET, req_id: 70454771-459b-4510-8e83-c2073313f6b7, remote_addr: 127.0.0.1:57841, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.779 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/077.copy" to "/tmp/downstairs-zrMnlo6G/00/000/077.replace"
Sep 22 23:22:20.779 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.780 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/077.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.780 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/077"
Sep 22 23:22:20.780 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/077.db"
Sep 22 23:22:20.781 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.781 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/077.replace" to "/tmp/downstairs-zrMnlo6G/00/000/077.completed"
Sep 22 23:22:20.781 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.781 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.781 DEBG [0] It's time to notify for 338
Sep 22 23:22:20.781 INFO Completion from [0] id:338 status:true
Sep 22 23:22:20.781 INFO [339/752] Repair commands completed
Sep 22 23:22:20.781 INFO Pop front: ReconcileIO { id: ReconciliationId(339), op: ExtentReopen { repair_id: ReconciliationId(339), extent_id: 119 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.781 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.781 INFO [0] received reconcile message
Sep 22 23:22:20.781 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(339), op: ExtentReopen { repair_id: ReconciliationId(339), extent_id: 119 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.781 INFO [0] client ExtentReopen { repair_id: ReconciliationId(339), extent_id: 119 }
Sep 22 23:22:20.781 INFO [1] received reconcile message
Sep 22 23:22:20.781 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(339), op: ExtentReopen { repair_id: ReconciliationId(339), extent_id: 119 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.781 INFO [1] client ExtentReopen { repair_id: ReconciliationId(339), extent_id: 119 }
Sep 22 23:22:20.781 INFO [2] received reconcile message
Sep 22 23:22:20.781 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(339), op: ExtentReopen { repair_id: ReconciliationId(339), extent_id: 119 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.781 INFO [2] client ExtentReopen { repair_id: ReconciliationId(339), extent_id: 119 }
Sep 22 23:22:20.781 DEBG 339 Reopen extent 119
Sep 22 23:22:20.782 DEBG 339 Reopen extent 119
Sep 22 23:22:20.783 DEBG 339 Reopen extent 119
Sep 22 23:22:20.783 DEBG [2] It's time to notify for 339
Sep 22 23:22:20.783 INFO Completion from [2] id:339 status:true
Sep 22 23:22:20.783 INFO [340/752] Repair commands completed
Sep 22 23:22:20.783 INFO Pop front: ReconcileIO { id: ReconciliationId(340), op: ExtentFlush { repair_id: ReconciliationId(340), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.783 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.783 INFO [0] received reconcile message
Sep 22 23:22:20.783 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(340), op: ExtentFlush { repair_id: ReconciliationId(340), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.783 INFO [0] client ExtentFlush { repair_id: ReconciliationId(340), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.783 INFO [1] received reconcile message
Sep 22 23:22:20.783 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(340), op: ExtentFlush { repair_id: ReconciliationId(340), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.783 INFO [1] client ExtentFlush { repair_id: ReconciliationId(340), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.783 INFO [2] received reconcile message
Sep 22 23:22:20.783 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(340), op: ExtentFlush { repair_id: ReconciliationId(340), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.783 INFO [2] client ExtentFlush { repair_id: ReconciliationId(340), extent_id: 30, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.784 DEBG 340 Flush extent 30 with f:2 g:2
Sep 22 23:22:20.784 DEBG Flush just extent 30 with f:2 and g:2
Sep 22 23:22:20.784 DEBG [1] It's time to notify for 340
Sep 22 23:22:20.784 INFO Completion from [1] id:340 status:true
Sep 22 23:22:20.784 INFO [341/752] Repair commands completed
Sep 22 23:22:20.784 INFO Pop front: ReconcileIO { id: ReconciliationId(341), op: ExtentClose { repair_id: ReconciliationId(341), extent_id: 30 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.784 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.784 INFO [0] received reconcile message
Sep 22 23:22:20.784 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(341), op: ExtentClose { repair_id: ReconciliationId(341), extent_id: 30 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.784 INFO [0] client ExtentClose { repair_id: ReconciliationId(341), extent_id: 30 }
Sep 22 23:22:20.784 INFO [1] received reconcile message
Sep 22 23:22:20.784 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(341), op: ExtentClose { repair_id: ReconciliationId(341), extent_id: 30 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.784 INFO [1] client ExtentClose { repair_id: ReconciliationId(341), extent_id: 30 }
Sep 22 23:22:20.784 INFO [2] received reconcile message
Sep 22 23:22:20.784 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(341), op: ExtentClose { repair_id: ReconciliationId(341), extent_id: 30 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.784 INFO [2] client ExtentClose { repair_id: ReconciliationId(341), extent_id: 30 }
Sep 22 23:22:20.784 DEBG 341 Close extent 30
Sep 22 23:22:20.784 DEBG 341 Close extent 30
Sep 22 23:22:20.785 DEBG 341 Close extent 30
Sep 22 23:22:20.785 DEBG [2] It's time to notify for 341
Sep 22 23:22:20.785 INFO Completion from [2] id:341 status:true
Sep 22 23:22:20.785 INFO [342/752] Repair commands completed
Sep 22 23:22:20.785 INFO Pop front: ReconcileIO { id: ReconciliationId(342), op: ExtentRepair { repair_id: ReconciliationId(342), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.785 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.785 INFO [0] received reconcile message
Sep 22 23:22:20.785 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(342), op: ExtentRepair { repair_id: ReconciliationId(342), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.785 INFO [0] client ExtentRepair { repair_id: ReconciliationId(342), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.785 INFO [0] Sending repair request ReconciliationId(342)
Sep 22 23:22:20.785 INFO [1] received reconcile message
Sep 22 23:22:20.785 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(342), op: ExtentRepair { repair_id: ReconciliationId(342), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.785 INFO [1] client ExtentRepair { repair_id: ReconciliationId(342), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.785 INFO [1] No action required ReconciliationId(342)
Sep 22 23:22:20.785 INFO [2] received reconcile message
Sep 22 23:22:20.785 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(342), op: ExtentRepair { repair_id: ReconciliationId(342), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.786 INFO [2] client ExtentRepair { repair_id: ReconciliationId(342), extent_id: 30, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.786 INFO [2] No action required ReconciliationId(342)
Sep 22 23:22:20.786 DEBG 342 Repair extent 30 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.786 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/01E.copy"
Sep 22 23:22:20.839 ERRO [2] job id 1081 saw error GenericError("test error")
Sep 22 23:22:20.844 DEBG up_ds_listen was notified
Sep 22 23:22:20.844 DEBG up_ds_listen process 1082
Sep 22 23:22:20.844 DEBG [A] ack job 1082:83, : downstairs
Sep 22 23:22:20.844 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:20.850 INFO accepted connection, remote_addr: 127.0.0.1:58426, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.850 TRCE incoming request, uri: /extent/30/files, method: GET, req_id: 42aedd9b-da3a-446d-afc9-ca44553ea593, remote_addr: 127.0.0.1:58426, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.850 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/30/files, method: GET, req_id: 42aedd9b-da3a-446d-afc9-ca44553ea593, remote_addr: 127.0.0.1:58426, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.850 INFO eid:30 Found repair files: ["01E", "01E.db"]
Sep 22 23:22:20.851 TRCE incoming request, uri: /newextent/30/data, method: GET, req_id: cd8bc92a-65d6-414e-81db-c90b8063e733, remote_addr: 127.0.0.1:58426, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.851 DEBG IO Flush 1084 has deps [JobId(1083), JobId(1082)]
Sep 22 23:22:20.851 INFO request completed, latency_us: 363, response_code: 200, uri: /newextent/30/data, method: GET, req_id: cd8bc92a-65d6-414e-81db-c90b8063e733, remote_addr: 127.0.0.1:58426, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.853 INFO [lossy] sleeping 1 second
Sep 22 23:22:20.856 TRCE incoming request, uri: /newextent/30/db, method: GET, req_id: 33618b72-82e5-4f7a-866f-ebebf093c319, remote_addr: 127.0.0.1:58426, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.856 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/30/db, method: GET, req_id: 33618b72-82e5-4f7a-866f-ebebf093c319, remote_addr: 127.0.0.1:58426, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.857 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/01E.copy" to "/tmp/downstairs-zrMnlo6G/00/000/01E.replace"
Sep 22 23:22:20.857 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.858 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/01E.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.858 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01E"
Sep 22 23:22:20.858 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01E.db"
Sep 22 23:22:20.859 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.859 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/01E.replace" to "/tmp/downstairs-zrMnlo6G/00/000/01E.completed"
Sep 22 23:22:20.859 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.859 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.859 DEBG [0] It's time to notify for 342
Sep 22 23:22:20.859 INFO Completion from [0] id:342 status:true
Sep 22 23:22:20.859 INFO [343/752] Repair commands completed
Sep 22 23:22:20.859 INFO Pop front: ReconcileIO { id: ReconciliationId(343), op: ExtentReopen { repair_id: ReconciliationId(343), extent_id: 30 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.859 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.859 INFO [0] received reconcile message
Sep 22 23:22:20.859 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(343), op: ExtentReopen { repair_id: ReconciliationId(343), extent_id: 30 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.859 INFO [0] client ExtentReopen { repair_id: ReconciliationId(343), extent_id: 30 }
Sep 22 23:22:20.859 INFO [1] received reconcile message
Sep 22 23:22:20.859 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(343), op: ExtentReopen { repair_id: ReconciliationId(343), extent_id: 30 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.859 INFO [1] client ExtentReopen { repair_id: ReconciliationId(343), extent_id: 30 }
Sep 22 23:22:20.859 INFO [2] received reconcile message
Sep 22 23:22:20.859 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(343), op: ExtentReopen { repair_id: ReconciliationId(343), extent_id: 30 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.859 INFO [2] client ExtentReopen { repair_id: ReconciliationId(343), extent_id: 30 }
Sep 22 23:22:20.859 DEBG 343 Reopen extent 30
Sep 22 23:22:20.860 DEBG 343 Reopen extent 30
Sep 22 23:22:20.861 DEBG 343 Reopen extent 30
Sep 22 23:22:20.861 DEBG [2] It's time to notify for 343
Sep 22 23:22:20.861 INFO Completion from [2] id:343 status:true
Sep 22 23:22:20.861 INFO [344/752] Repair commands completed
Sep 22 23:22:20.861 INFO Pop front: ReconcileIO { id: ReconciliationId(344), op: ExtentFlush { repair_id: ReconciliationId(344), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.861 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.861 INFO [0] received reconcile message
Sep 22 23:22:20.861 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(344), op: ExtentFlush { repair_id: ReconciliationId(344), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.861 INFO [0] client ExtentFlush { repair_id: ReconciliationId(344), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.861 INFO [1] received reconcile message
Sep 22 23:22:20.861 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(344), op: ExtentFlush { repair_id: ReconciliationId(344), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.861 INFO [1] client ExtentFlush { repair_id: ReconciliationId(344), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.861 INFO [2] received reconcile message
Sep 22 23:22:20.861 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(344), op: ExtentFlush { repair_id: ReconciliationId(344), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.861 INFO [2] client ExtentFlush { repair_id: ReconciliationId(344), extent_id: 46, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.862 DEBG 344 Flush extent 46 with f:2 g:2
Sep 22 23:22:20.862 DEBG Flush just extent 46 with f:2 and g:2
Sep 22 23:22:20.862 DEBG [1] It's time to notify for 344
Sep 22 23:22:20.862 INFO Completion from [1] id:344 status:true
Sep 22 23:22:20.862 INFO [345/752] Repair commands completed
Sep 22 23:22:20.862 INFO Pop front: ReconcileIO { id: ReconciliationId(345), op: ExtentClose { repair_id: ReconciliationId(345), extent_id: 46 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.862 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.862 INFO [0] received reconcile message
Sep 22 23:22:20.862 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(345), op: ExtentClose { repair_id: ReconciliationId(345), extent_id: 46 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.862 INFO [0] client ExtentClose { repair_id: ReconciliationId(345), extent_id: 46 }
Sep 22 23:22:20.862 INFO [1] received reconcile message
Sep 22 23:22:20.862 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(345), op: ExtentClose { repair_id: ReconciliationId(345), extent_id: 46 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.862 INFO [1] client ExtentClose { repair_id: ReconciliationId(345), extent_id: 46 }
Sep 22 23:22:20.862 INFO [2] received reconcile message
Sep 22 23:22:20.862 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(345), op: ExtentClose { repair_id: ReconciliationId(345), extent_id: 46 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.862 INFO [2] client ExtentClose { repair_id: ReconciliationId(345), extent_id: 46 }
Sep 22 23:22:20.862 DEBG 345 Close extent 46
Sep 22 23:22:20.862 DEBG 345 Close extent 46
Sep 22 23:22:20.863 DEBG 345 Close extent 46
Sep 22 23:22:20.863 DEBG [2] It's time to notify for 345
Sep 22 23:22:20.863 INFO Completion from [2] id:345 status:true
Sep 22 23:22:20.863 INFO [346/752] Repair commands completed
Sep 22 23:22:20.863 INFO Pop front: ReconcileIO { id: ReconciliationId(346), op: ExtentRepair { repair_id: ReconciliationId(346), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.863 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.863 INFO [0] received reconcile message
Sep 22 23:22:20.863 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(346), op: ExtentRepair { repair_id: ReconciliationId(346), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.863 INFO [0] client ExtentRepair { repair_id: ReconciliationId(346), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.863 INFO [0] Sending repair request ReconciliationId(346)
Sep 22 23:22:20.863 INFO [1] received reconcile message
Sep 22 23:22:20.863 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(346), op: ExtentRepair { repair_id: ReconciliationId(346), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.863 INFO [1] client ExtentRepair { repair_id: ReconciliationId(346), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.863 INFO [1] No action required ReconciliationId(346)
Sep 22 23:22:20.863 INFO [2] received reconcile message
Sep 22 23:22:20.863 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(346), op: ExtentRepair { repair_id: ReconciliationId(346), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.864 INFO [2] client ExtentRepair { repair_id: ReconciliationId(346), extent_id: 46, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.864 INFO [2] No action required ReconciliationId(346)
Sep 22 23:22:20.864 DEBG 346 Repair extent 46 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.864 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/02E.copy"
Sep 22 23:22:20.930 INFO accepted connection, remote_addr: 127.0.0.1:62231, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.930 TRCE incoming request, uri: /extent/46/files, method: GET, req_id: 52fa3f60-9101-408a-ae8c-2e8375d14443, remote_addr: 127.0.0.1:62231, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.930 INFO request completed, latency_us: 276, response_code: 200, uri: /extent/46/files, method: GET, req_id: 52fa3f60-9101-408a-ae8c-2e8375d14443, remote_addr: 127.0.0.1:62231, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.930 INFO eid:46 Found repair files: ["02E", "02E.db"]
Sep 22 23:22:20.931 TRCE incoming request, uri: /newextent/46/data, method: GET, req_id: d134f810-e211-4f22-a384-f33489afcd36, remote_addr: 127.0.0.1:62231, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.931 INFO request completed, latency_us: 342, response_code: 200, uri: /newextent/46/data, method: GET, req_id: d134f810-e211-4f22-a384-f33489afcd36, remote_addr: 127.0.0.1:62231, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.936 TRCE incoming request, uri: /newextent/46/db, method: GET, req_id: 31ff714d-d9b8-425c-8cf8-0c0b8301bc57, remote_addr: 127.0.0.1:62231, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.936 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/46/db, method: GET, req_id: 31ff714d-d9b8-425c-8cf8-0c0b8301bc57, remote_addr: 127.0.0.1:62231, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:20.938 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/02E.copy" to "/tmp/downstairs-zrMnlo6G/00/000/02E.replace"
Sep 22 23:22:20.938 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.939 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/02E.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.939 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02E"
Sep 22 23:22:20.939 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02E.db"
Sep 22 23:22:20.939 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.939 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/02E.replace" to "/tmp/downstairs-zrMnlo6G/00/000/02E.completed"
Sep 22 23:22:20.939 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.939 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:20.940 DEBG [0] It's time to notify for 346
Sep 22 23:22:20.940 INFO Completion from [0] id:346 status:true
Sep 22 23:22:20.940 INFO [347/752] Repair commands completed
Sep 22 23:22:20.940 INFO Pop front: ReconcileIO { id: ReconciliationId(347), op: ExtentReopen { repair_id: ReconciliationId(347), extent_id: 46 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.940 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.940 INFO [0] received reconcile message
Sep 22 23:22:20.940 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(347), op: ExtentReopen { repair_id: ReconciliationId(347), extent_id: 46 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.940 INFO [0] client ExtentReopen { repair_id: ReconciliationId(347), extent_id: 46 }
Sep 22 23:22:20.940 INFO [1] received reconcile message
Sep 22 23:22:20.940 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(347), op: ExtentReopen { repair_id: ReconciliationId(347), extent_id: 46 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.940 INFO [1] client ExtentReopen { repair_id: ReconciliationId(347), extent_id: 46 }
Sep 22 23:22:20.940 INFO [2] received reconcile message
Sep 22 23:22:20.940 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(347), op: ExtentReopen { repair_id: ReconciliationId(347), extent_id: 46 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.940 INFO [2] client ExtentReopen { repair_id: ReconciliationId(347), extent_id: 46 }
Sep 22 23:22:20.940 DEBG 347 Reopen extent 46
Sep 22 23:22:20.941 DEBG 347 Reopen extent 46
Sep 22 23:22:20.942 DEBG 347 Reopen extent 46
Sep 22 23:22:20.942 DEBG [2] It's time to notify for 347
Sep 22 23:22:20.942 INFO Completion from [2] id:347 status:true
Sep 22 23:22:20.942 INFO [348/752] Repair commands completed
Sep 22 23:22:20.942 INFO Pop front: ReconcileIO { id: ReconciliationId(348), op: ExtentFlush { repair_id: ReconciliationId(348), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.942 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.943 INFO [0] received reconcile message
Sep 22 23:22:20.943 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(348), op: ExtentFlush { repair_id: ReconciliationId(348), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.943 INFO [0] client ExtentFlush { repair_id: ReconciliationId(348), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.943 INFO [1] received reconcile message
Sep 22 23:22:20.943 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(348), op: ExtentFlush { repair_id: ReconciliationId(348), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:20.943 INFO [1] client ExtentFlush { repair_id: ReconciliationId(348), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.943 INFO [2] received reconcile message
Sep 22 23:22:20.943 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(348), op: ExtentFlush { repair_id: ReconciliationId(348), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.943 INFO [2] client ExtentFlush { repair_id: ReconciliationId(348), extent_id: 127, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:20.943 DEBG 348 Flush extent 127 with f:2 g:2
Sep 22 23:22:20.943 DEBG Flush just extent 127 with f:2 and g:2
Sep 22 23:22:20.943 DEBG [1] It's time to notify for 348
Sep 22 23:22:20.943 INFO Completion from [1] id:348 status:true
Sep 22 23:22:20.943 INFO [349/752] Repair commands completed
Sep 22 23:22:20.943 INFO Pop front: ReconcileIO { id: ReconciliationId(349), op: ExtentClose { repair_id: ReconciliationId(349), extent_id: 127 }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.943 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.943 INFO [0] received reconcile message
Sep 22 23:22:20.943 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(349), op: ExtentClose { repair_id: ReconciliationId(349), extent_id: 127 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.943 INFO [0] client ExtentClose { repair_id: ReconciliationId(349), extent_id: 127 }
Sep 22 23:22:20.943 INFO [1] received reconcile message
Sep 22 23:22:20.943 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(349), op: ExtentClose { repair_id: ReconciliationId(349), extent_id: 127 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.943 INFO [1] client ExtentClose { repair_id: ReconciliationId(349), extent_id: 127 }
Sep 22 23:22:20.943 INFO [2] received reconcile message
Sep 22 23:22:20.943 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(349), op: ExtentClose { repair_id: ReconciliationId(349), extent_id: 127 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:20.943 INFO [2] client ExtentClose { repair_id: ReconciliationId(349), extent_id: 127 }
Sep 22 23:22:20.943 DEBG 349 Close extent 127
Sep 22 23:22:20.944 DEBG 349 Close extent 127
Sep 22 23:22:20.944 DEBG 349 Close extent 127
Sep 22 23:22:20.944 DEBG [2] It's time to notify for 349
Sep 22 23:22:20.944 INFO Completion from [2] id:349 status:true
Sep 22 23:22:20.944 INFO [350/752] Repair commands completed
Sep 22 23:22:20.944 INFO Pop front: ReconcileIO { id: ReconciliationId(350), op: ExtentRepair { repair_id: ReconciliationId(350), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:20.945 INFO Sent repair work, now wait for resp
Sep 22 23:22:20.945 INFO [0] received reconcile message
Sep 22 23:22:20.945 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(350), op: ExtentRepair { repair_id: ReconciliationId(350), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:20.945 INFO [0] client ExtentRepair { repair_id: ReconciliationId(350), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.945 INFO [0] Sending repair request ReconciliationId(350)
Sep 22 23:22:20.945 INFO [1] received reconcile message
Sep 22 23:22:20.945 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(350), op: ExtentRepair { repair_id: ReconciliationId(350), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:20.945 INFO [1] client ExtentRepair { repair_id: ReconciliationId(350), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.945 INFO [1] No action required ReconciliationId(350)
Sep 22 23:22:20.945 INFO [2] received reconcile message
Sep 22 23:22:20.945 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(350), op: ExtentRepair { repair_id: ReconciliationId(350), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:20.945 INFO [2] client ExtentRepair { repair_id: ReconciliationId(350), extent_id: 127, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:20.945 INFO [2] No action required ReconciliationId(350)
Sep 22 23:22:20.945 DEBG 350 Repair extent 127 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:20.945 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/07F.copy"
Sep 22 23:22:21.011 INFO accepted connection, remote_addr: 127.0.0.1:56439, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.012 TRCE incoming request, uri: /extent/127/files, method: GET, req_id: 46989101-851b-4e66-b25a-724ac843852a, remote_addr: 127.0.0.1:56439, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.012 INFO request completed, latency_us: 296, response_code: 200, uri: /extent/127/files, method: GET, req_id: 46989101-851b-4e66-b25a-724ac843852a, remote_addr: 127.0.0.1:56439, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.012 INFO eid:127 Found repair files: ["07F", "07F.db"]
Sep 22 23:22:21.013 TRCE incoming request, uri: /newextent/127/data, method: GET, req_id: eed44035-bfdb-46b8-85e3-330fb507ba9a, remote_addr: 127.0.0.1:56439, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.013 INFO request completed, latency_us: 359, response_code: 200, uri: /newextent/127/data, method: GET, req_id: eed44035-bfdb-46b8-85e3-330fb507ba9a, remote_addr: 127.0.0.1:56439, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.018 TRCE incoming request, uri: /newextent/127/db, method: GET, req_id: 7260fb59-b8f9-4d8c-84e3-237cc7565e46, remote_addr: 127.0.0.1:56439, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.018 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/127/db, method: GET, req_id: 7260fb59-b8f9-4d8c-84e3-237cc7565e46, remote_addr: 127.0.0.1:56439, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.019 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/07F.copy" to "/tmp/downstairs-zrMnlo6G/00/000/07F.replace"
Sep 22 23:22:21.019 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.020 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/07F.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.021 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07F"
Sep 22 23:22:21.021 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07F.db"
Sep 22 23:22:21.021 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.021 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/07F.replace" to "/tmp/downstairs-zrMnlo6G/00/000/07F.completed"
Sep 22 23:22:21.021 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.021 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.021 DEBG [0] It's time to notify for 350
Sep 22 23:22:21.021 INFO Completion from [0] id:350 status:true
Sep 22 23:22:21.021 INFO [351/752] Repair commands completed
Sep 22 23:22:21.021 INFO Pop front: ReconcileIO { id: ReconciliationId(351), op: ExtentReopen { repair_id: ReconciliationId(351), extent_id: 127 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.021 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.021 INFO [0] received reconcile message
Sep 22 23:22:21.021 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(351), op: ExtentReopen { repair_id: ReconciliationId(351), extent_id: 127 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.022 INFO [0] client ExtentReopen { repair_id: ReconciliationId(351), extent_id: 127 }
Sep 22 23:22:21.022 INFO [1] received reconcile message
Sep 22 23:22:21.022 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(351), op: ExtentReopen { repair_id: ReconciliationId(351), extent_id: 127 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.022 INFO [1] client ExtentReopen { repair_id: ReconciliationId(351), extent_id: 127 }
Sep 22 23:22:21.022 INFO [2] received reconcile message
Sep 22 23:22:21.022 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(351), op: ExtentReopen { repair_id: ReconciliationId(351), extent_id: 127 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.022 INFO [2] client ExtentReopen { repair_id: ReconciliationId(351), extent_id: 127 }
Sep 22 23:22:21.022 DEBG 351 Reopen extent 127
Sep 22 23:22:21.023 DEBG 351 Reopen extent 127
Sep 22 23:22:21.023 DEBG 351 Reopen extent 127
Sep 22 23:22:21.024 DEBG [2] It's time to notify for 351
Sep 22 23:22:21.024 INFO Completion from [2] id:351 status:true
Sep 22 23:22:21.024 INFO [352/752] Repair commands completed
Sep 22 23:22:21.024 INFO Pop front: ReconcileIO { id: ReconciliationId(352), op: ExtentFlush { repair_id: ReconciliationId(352), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.024 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.024 INFO [0] received reconcile message
Sep 22 23:22:21.024 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(352), op: ExtentFlush { repair_id: ReconciliationId(352), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.024 INFO [0] client ExtentFlush { repair_id: ReconciliationId(352), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.024 INFO [1] received reconcile message
Sep 22 23:22:21.024 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(352), op: ExtentFlush { repair_id: ReconciliationId(352), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.024 INFO [1] client ExtentFlush { repair_id: ReconciliationId(352), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.024 INFO [2] received reconcile message
Sep 22 23:22:21.024 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(352), op: ExtentFlush { repair_id: ReconciliationId(352), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.024 INFO [2] client ExtentFlush { repair_id: ReconciliationId(352), extent_id: 99, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.024 DEBG 352 Flush extent 99 with f:2 g:2
Sep 22 23:22:21.024 DEBG Flush just extent 99 with f:2 and g:2
Sep 22 23:22:21.024 DEBG [1] It's time to notify for 352
Sep 22 23:22:21.024 INFO Completion from [1] id:352 status:true
Sep 22 23:22:21.024 INFO [353/752] Repair commands completed
Sep 22 23:22:21.024 INFO Pop front: ReconcileIO { id: ReconciliationId(353), op: ExtentClose { repair_id: ReconciliationId(353), extent_id: 99 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.025 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.025 INFO [0] received reconcile message
Sep 22 23:22:21.025 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(353), op: ExtentClose { repair_id: ReconciliationId(353), extent_id: 99 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.025 INFO [0] client ExtentClose { repair_id: ReconciliationId(353), extent_id: 99 }
Sep 22 23:22:21.025 INFO [1] received reconcile message
Sep 22 23:22:21.025 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(353), op: ExtentClose { repair_id: ReconciliationId(353), extent_id: 99 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.025 INFO [1] client ExtentClose { repair_id: ReconciliationId(353), extent_id: 99 }
Sep 22 23:22:21.025 INFO [2] received reconcile message
Sep 22 23:22:21.025 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(353), op: ExtentClose { repair_id: ReconciliationId(353), extent_id: 99 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.025 INFO [2] client ExtentClose { repair_id: ReconciliationId(353), extent_id: 99 }
Sep 22 23:22:21.025 DEBG 353 Close extent 99
Sep 22 23:22:21.025 DEBG 353 Close extent 99
Sep 22 23:22:21.025 DEBG 353 Close extent 99
Sep 22 23:22:21.026 DEBG [2] It's time to notify for 353
Sep 22 23:22:21.026 INFO Completion from [2] id:353 status:true
Sep 22 23:22:21.026 INFO [354/752] Repair commands completed
Sep 22 23:22:21.026 INFO Pop front: ReconcileIO { id: ReconciliationId(354), op: ExtentRepair { repair_id: ReconciliationId(354), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.026 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.026 INFO [0] received reconcile message
Sep 22 23:22:21.026 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(354), op: ExtentRepair { repair_id: ReconciliationId(354), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.026 INFO [0] client ExtentRepair { repair_id: ReconciliationId(354), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.026 INFO [0] Sending repair request ReconciliationId(354)
Sep 22 23:22:21.026 INFO [1] received reconcile message
Sep 22 23:22:21.026 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(354), op: ExtentRepair { repair_id: ReconciliationId(354), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.026 INFO [1] client ExtentRepair { repair_id: ReconciliationId(354), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.026 INFO [1] No action required ReconciliationId(354)
Sep 22 23:22:21.026 INFO [2] received reconcile message
Sep 22 23:22:21.026 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(354), op: ExtentRepair { repair_id: ReconciliationId(354), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.026 INFO [2] client ExtentRepair { repair_id: ReconciliationId(354), extent_id: 99, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.026 INFO [2] No action required ReconciliationId(354)
Sep 22 23:22:21.026 DEBG 354 Repair extent 99 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.026 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/063.copy"
Sep 22 23:22:21.092 INFO accepted connection, remote_addr: 127.0.0.1:47636, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.092 TRCE incoming request, uri: /extent/99/files, method: GET, req_id: fd1881d2-5a5b-43ae-aff7-5d1b019d18d8, remote_addr: 127.0.0.1:47636, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.093 INFO request completed, latency_us: 223, response_code: 200, uri: /extent/99/files, method: GET, req_id: fd1881d2-5a5b-43ae-aff7-5d1b019d18d8, remote_addr: 127.0.0.1:47636, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.093 INFO eid:99 Found repair files: ["063", "063.db"]
Sep 22 23:22:21.093 TRCE incoming request, uri: /newextent/99/data, method: GET, req_id: e51b3f22-3657-4d69-a52c-f88e0b77ad75, remote_addr: 127.0.0.1:47636, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.094 INFO request completed, latency_us: 318, response_code: 200, uri: /newextent/99/data, method: GET, req_id: e51b3f22-3657-4d69-a52c-f88e0b77ad75, remote_addr: 127.0.0.1:47636, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.099 TRCE incoming request, uri: /newextent/99/db, method: GET, req_id: 9f83f553-3592-41b1-be69-c50140905ff2, remote_addr: 127.0.0.1:47636, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.099 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/99/db, method: GET, req_id: 9f83f553-3592-41b1-be69-c50140905ff2, remote_addr: 127.0.0.1:47636, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.100 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/063.copy" to "/tmp/downstairs-zrMnlo6G/00/000/063.replace"
Sep 22 23:22:21.100 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.101 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/063.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.101 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/063"
Sep 22 23:22:21.101 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/063.db"
Sep 22 23:22:21.101 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.101 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/063.replace" to "/tmp/downstairs-zrMnlo6G/00/000/063.completed"
Sep 22 23:22:21.102 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.102 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.102 DEBG [0] It's time to notify for 354
Sep 22 23:22:21.102 INFO Completion from [0] id:354 status:true
Sep 22 23:22:21.102 INFO [355/752] Repair commands completed
Sep 22 23:22:21.102 INFO Pop front: ReconcileIO { id: ReconciliationId(355), op: ExtentReopen { repair_id: ReconciliationId(355), extent_id: 99 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.102 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.102 INFO [0] received reconcile message
Sep 22 23:22:21.102 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(355), op: ExtentReopen { repair_id: ReconciliationId(355), extent_id: 99 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.102 INFO [0] client ExtentReopen { repair_id: ReconciliationId(355), extent_id: 99 }
Sep 22 23:22:21.102 INFO [1] received reconcile message
Sep 22 23:22:21.102 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(355), op: ExtentReopen { repair_id: ReconciliationId(355), extent_id: 99 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.102 INFO [1] client ExtentReopen { repair_id: ReconciliationId(355), extent_id: 99 }
Sep 22 23:22:21.102 INFO [2] received reconcile message
Sep 22 23:22:21.102 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(355), op: ExtentReopen { repair_id: ReconciliationId(355), extent_id: 99 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.102 INFO [2] client ExtentReopen { repair_id: ReconciliationId(355), extent_id: 99 }
Sep 22 23:22:21.102 DEBG 355 Reopen extent 99
Sep 22 23:22:21.103 DEBG 355 Reopen extent 99
Sep 22 23:22:21.104 DEBG 355 Reopen extent 99
Sep 22 23:22:21.104 DEBG [2] It's time to notify for 355
Sep 22 23:22:21.104 INFO Completion from [2] id:355 status:true
Sep 22 23:22:21.104 INFO [356/752] Repair commands completed
Sep 22 23:22:21.104 INFO Pop front: ReconcileIO { id: ReconciliationId(356), op: ExtentFlush { repair_id: ReconciliationId(356), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.104 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.104 INFO [0] received reconcile message
Sep 22 23:22:21.104 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(356), op: ExtentFlush { repair_id: ReconciliationId(356), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.104 INFO [0] client ExtentFlush { repair_id: ReconciliationId(356), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.104 INFO [1] received reconcile message
Sep 22 23:22:21.104 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(356), op: ExtentFlush { repair_id: ReconciliationId(356), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.104 INFO [1] client ExtentFlush { repair_id: ReconciliationId(356), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.105 INFO [2] received reconcile message
Sep 22 23:22:21.105 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(356), op: ExtentFlush { repair_id: ReconciliationId(356), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.105 INFO [2] client ExtentFlush { repair_id: ReconciliationId(356), extent_id: 12, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.105 DEBG 356 Flush extent 12 with f:2 g:2
Sep 22 23:22:21.105 DEBG Flush just extent 12 with f:2 and g:2
Sep 22 23:22:21.105 DEBG [1] It's time to notify for 356
Sep 22 23:22:21.105 INFO Completion from [1] id:356 status:true
Sep 22 23:22:21.105 INFO [357/752] Repair commands completed
Sep 22 23:22:21.105 INFO Pop front: ReconcileIO { id: ReconciliationId(357), op: ExtentClose { repair_id: ReconciliationId(357), extent_id: 12 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.105 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.105 INFO [0] received reconcile message
Sep 22 23:22:21.105 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(357), op: ExtentClose { repair_id: ReconciliationId(357), extent_id: 12 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.105 INFO [0] client ExtentClose { repair_id: ReconciliationId(357), extent_id: 12 }
Sep 22 23:22:21.105 INFO [1] received reconcile message
Sep 22 23:22:21.105 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(357), op: ExtentClose { repair_id: ReconciliationId(357), extent_id: 12 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.105 INFO [1] client ExtentClose { repair_id: ReconciliationId(357), extent_id: 12 }
Sep 22 23:22:21.105 INFO [2] received reconcile message
Sep 22 23:22:21.105 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(357), op: ExtentClose { repair_id: ReconciliationId(357), extent_id: 12 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.105 INFO [2] client ExtentClose { repair_id: ReconciliationId(357), extent_id: 12 }
Sep 22 23:22:21.105 DEBG 357 Close extent 12
Sep 22 23:22:21.105 DEBG 357 Close extent 12
Sep 22 23:22:21.106 DEBG 357 Close extent 12
Sep 22 23:22:21.106 DEBG [2] It's time to notify for 357
Sep 22 23:22:21.106 INFO Completion from [2] id:357 status:true
Sep 22 23:22:21.106 INFO [358/752] Repair commands completed
Sep 22 23:22:21.106 INFO Pop front: ReconcileIO { id: ReconciliationId(358), op: ExtentRepair { repair_id: ReconciliationId(358), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.106 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.106 INFO [0] received reconcile message
Sep 22 23:22:21.106 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(358), op: ExtentRepair { repair_id: ReconciliationId(358), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.106 INFO [0] client ExtentRepair { repair_id: ReconciliationId(358), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.106 INFO [0] Sending repair request ReconciliationId(358)
Sep 22 23:22:21.106 INFO [1] received reconcile message
Sep 22 23:22:21.106 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(358), op: ExtentRepair { repair_id: ReconciliationId(358), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.107 INFO [1] client ExtentRepair { repair_id: ReconciliationId(358), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.107 INFO [1] No action required ReconciliationId(358)
Sep 22 23:22:21.107 INFO [2] received reconcile message
Sep 22 23:22:21.107 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(358), op: ExtentRepair { repair_id: ReconciliationId(358), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.107 INFO [2] client ExtentRepair { repair_id: ReconciliationId(358), extent_id: 12, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.107 INFO [2] No action required ReconciliationId(358)
Sep 22 23:22:21.107 DEBG 358 Repair extent 12 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.107 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/00C.copy"
Sep 22 23:22:21.172 INFO accepted connection, remote_addr: 127.0.0.1:64228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.173 TRCE incoming request, uri: /extent/12/files, method: GET, req_id: b3c02c20-cc84-46c0-9438-3e2db2f72443, remote_addr: 127.0.0.1:64228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.173 INFO request completed, latency_us: 256, response_code: 200, uri: /extent/12/files, method: GET, req_id: b3c02c20-cc84-46c0-9438-3e2db2f72443, remote_addr: 127.0.0.1:64228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.173 INFO eid:12 Found repair files: ["00C", "00C.db"]
Sep 22 23:22:21.174 TRCE incoming request, uri: /newextent/12/data, method: GET, req_id: 6fe9f463-2599-48c2-9b27-82af677233e0, remote_addr: 127.0.0.1:64228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.174 INFO request completed, latency_us: 342, response_code: 200, uri: /newextent/12/data, method: GET, req_id: 6fe9f463-2599-48c2-9b27-82af677233e0, remote_addr: 127.0.0.1:64228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.179 TRCE incoming request, uri: /newextent/12/db, method: GET, req_id: 8e04cb31-5a20-4989-8458-bc37e9e1d510, remote_addr: 127.0.0.1:64228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.179 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/12/db, method: GET, req_id: 8e04cb31-5a20-4989-8458-bc37e9e1d510, remote_addr: 127.0.0.1:64228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.180 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/00C.copy" to "/tmp/downstairs-zrMnlo6G/00/000/00C.replace"
Sep 22 23:22:21.180 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.182 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/00C.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.182 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00C"
Sep 22 23:22:21.182 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00C.db"
Sep 22 23:22:21.182 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.182 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/00C.replace" to "/tmp/downstairs-zrMnlo6G/00/000/00C.completed"
Sep 22 23:22:21.182 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.182 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.182 DEBG [0] It's time to notify for 358
Sep 22 23:22:21.182 INFO Completion from [0] id:358 status:true
Sep 22 23:22:21.182 INFO [359/752] Repair commands completed
Sep 22 23:22:21.182 INFO Pop front: ReconcileIO { id: ReconciliationId(359), op: ExtentReopen { repair_id: ReconciliationId(359), extent_id: 12 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.182 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.183 INFO [0] received reconcile message
Sep 22 23:22:21.183 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(359), op: ExtentReopen { repair_id: ReconciliationId(359), extent_id: 12 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.183 INFO [0] client ExtentReopen { repair_id: ReconciliationId(359), extent_id: 12 }
Sep 22 23:22:21.183 INFO [1] received reconcile message
Sep 22 23:22:21.183 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(359), op: ExtentReopen { repair_id: ReconciliationId(359), extent_id: 12 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.183 INFO [1] client ExtentReopen { repair_id: ReconciliationId(359), extent_id: 12 }
Sep 22 23:22:21.183 INFO [2] received reconcile message
Sep 22 23:22:21.183 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(359), op: ExtentReopen { repair_id: ReconciliationId(359), extent_id: 12 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.183 INFO [2] client ExtentReopen { repair_id: ReconciliationId(359), extent_id: 12 }
Sep 22 23:22:21.183 DEBG 359 Reopen extent 12
Sep 22 23:22:21.184 DEBG 359 Reopen extent 12
Sep 22 23:22:21.184 DEBG 359 Reopen extent 12
Sep 22 23:22:21.185 DEBG [2] It's time to notify for 359
Sep 22 23:22:21.185 INFO Completion from [2] id:359 status:true
Sep 22 23:22:21.185 INFO [360/752] Repair commands completed
Sep 22 23:22:21.185 INFO Pop front: ReconcileIO { id: ReconciliationId(360), op: ExtentFlush { repair_id: ReconciliationId(360), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.185 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.185 INFO [0] received reconcile message
Sep 22 23:22:21.185 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(360), op: ExtentFlush { repair_id: ReconciliationId(360), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.185 INFO [0] client ExtentFlush { repair_id: ReconciliationId(360), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.185 INFO [1] received reconcile message
Sep 22 23:22:21.185 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(360), op: ExtentFlush { repair_id: ReconciliationId(360), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.185 INFO [1] client ExtentFlush { repair_id: ReconciliationId(360), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.185 INFO [2] received reconcile message
Sep 22 23:22:21.185 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(360), op: ExtentFlush { repair_id: ReconciliationId(360), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.185 INFO [2] client ExtentFlush { repair_id: ReconciliationId(360), extent_id: 34, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.185 DEBG 360 Flush extent 34 with f:2 g:2
Sep 22 23:22:21.185 DEBG Flush just extent 34 with f:2 and g:2
Sep 22 23:22:21.185 DEBG [1] It's time to notify for 360
Sep 22 23:22:21.185 INFO Completion from [1] id:360 status:true
Sep 22 23:22:21.185 INFO [361/752] Repair commands completed
Sep 22 23:22:21.185 INFO Pop front: ReconcileIO { id: ReconciliationId(361), op: ExtentClose { repair_id: ReconciliationId(361), extent_id: 34 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.186 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.186 INFO [0] received reconcile message
Sep 22 23:22:21.186 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(361), op: ExtentClose { repair_id: ReconciliationId(361), extent_id: 34 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.186 INFO [0] client ExtentClose { repair_id: ReconciliationId(361), extent_id: 34 }
Sep 22 23:22:21.186 INFO [1] received reconcile message
Sep 22 23:22:21.186 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(361), op: ExtentClose { repair_id: ReconciliationId(361), extent_id: 34 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.186 INFO [1] client ExtentClose { repair_id: ReconciliationId(361), extent_id: 34 }
Sep 22 23:22:21.186 INFO [2] received reconcile message
Sep 22 23:22:21.186 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(361), op: ExtentClose { repair_id: ReconciliationId(361), extent_id: 34 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.186 INFO [2] client ExtentClose { repair_id: ReconciliationId(361), extent_id: 34 }
Sep 22 23:22:21.186 DEBG 361 Close extent 34
Sep 22 23:22:21.186 DEBG 361 Close extent 34
Sep 22 23:22:21.186 DEBG 361 Close extent 34
Sep 22 23:22:21.187 DEBG [2] It's time to notify for 361
Sep 22 23:22:21.187 INFO Completion from [2] id:361 status:true
Sep 22 23:22:21.187 INFO [362/752] Repair commands completed
Sep 22 23:22:21.187 INFO Pop front: ReconcileIO { id: ReconciliationId(362), op: ExtentRepair { repair_id: ReconciliationId(362), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.187 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.187 INFO [0] received reconcile message
Sep 22 23:22:21.187 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(362), op: ExtentRepair { repair_id: ReconciliationId(362), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.187 INFO [0] client ExtentRepair { repair_id: ReconciliationId(362), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.187 INFO [0] Sending repair request ReconciliationId(362)
Sep 22 23:22:21.187 INFO [1] received reconcile message
Sep 22 23:22:21.187 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(362), op: ExtentRepair { repair_id: ReconciliationId(362), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.187 INFO [1] client ExtentRepair { repair_id: ReconciliationId(362), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.187 INFO [1] No action required ReconciliationId(362)
Sep 22 23:22:21.187 INFO [2] received reconcile message
Sep 22 23:22:21.187 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(362), op: ExtentRepair { repair_id: ReconciliationId(362), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.187 INFO [2] client ExtentRepair { repair_id: ReconciliationId(362), extent_id: 34, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.187 INFO [2] No action required ReconciliationId(362)
Sep 22 23:22:21.187 DEBG 362 Repair extent 34 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.187 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/022.copy"
Sep 22 23:22:21.236 INFO [lossy] skipping 1083
Sep 22 23:22:21.236 INFO [lossy] skipping 1084
Sep 22 23:22:21.242 DEBG Read      :1083 deps:[JobId(1082)] res:true
Sep 22 23:22:21.249 INFO accepted connection, remote_addr: 127.0.0.1:62310, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.250 TRCE incoming request, uri: /extent/34/files, method: GET, req_id: 58fdb95c-fe9a-4eb3-a77d-4c3394e80c0c, remote_addr: 127.0.0.1:62310, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.250 INFO request completed, latency_us: 223, response_code: 200, uri: /extent/34/files, method: GET, req_id: 58fdb95c-fe9a-4eb3-a77d-4c3394e80c0c, remote_addr: 127.0.0.1:62310, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.250 INFO eid:34 Found repair files: ["022", "022.db"]
Sep 22 23:22:21.250 TRCE incoming request, uri: /newextent/34/data, method: GET, req_id: e0fa8d5c-95c4-41e9-a2cb-16171c25b105, remote_addr: 127.0.0.1:62310, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.251 INFO request completed, latency_us: 330, response_code: 200, uri: /newextent/34/data, method: GET, req_id: e0fa8d5c-95c4-41e9-a2cb-16171c25b105, remote_addr: 127.0.0.1:62310, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.256 TRCE incoming request, uri: /newextent/34/db, method: GET, req_id: 7122403c-e19c-4e1b-9ace-63fb2bc8a134, remote_addr: 127.0.0.1:62310, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.256 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/34/db, method: GET, req_id: 7122403c-e19c-4e1b-9ace-63fb2bc8a134, remote_addr: 127.0.0.1:62310, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.257 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/022.copy" to "/tmp/downstairs-zrMnlo6G/00/000/022.replace"
Sep 22 23:22:21.257 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.258 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/022.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.258 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/022"
Sep 22 23:22:21.258 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/022.db"
Sep 22 23:22:21.258 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.258 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/022.replace" to "/tmp/downstairs-zrMnlo6G/00/000/022.completed"
Sep 22 23:22:21.258 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.259 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.259 DEBG [0] It's time to notify for 362
Sep 22 23:22:21.259 INFO Completion from [0] id:362 status:true
Sep 22 23:22:21.259 INFO [363/752] Repair commands completed
Sep 22 23:22:21.259 INFO Pop front: ReconcileIO { id: ReconciliationId(363), op: ExtentReopen { repair_id: ReconciliationId(363), extent_id: 34 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.259 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.259 INFO [0] received reconcile message
Sep 22 23:22:21.259 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(363), op: ExtentReopen { repair_id: ReconciliationId(363), extent_id: 34 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.259 INFO [0] client ExtentReopen { repair_id: ReconciliationId(363), extent_id: 34 }
Sep 22 23:22:21.259 INFO [1] received reconcile message
Sep 22 23:22:21.259 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(363), op: ExtentReopen { repair_id: ReconciliationId(363), extent_id: 34 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.259 INFO [1] client ExtentReopen { repair_id: ReconciliationId(363), extent_id: 34 }
Sep 22 23:22:21.259 INFO [2] received reconcile message
Sep 22 23:22:21.259 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(363), op: ExtentReopen { repair_id: ReconciliationId(363), extent_id: 34 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.259 INFO [2] client ExtentReopen { repair_id: ReconciliationId(363), extent_id: 34 }
Sep 22 23:22:21.259 DEBG 363 Reopen extent 34
Sep 22 23:22:21.260 DEBG 363 Reopen extent 34
Sep 22 23:22:21.261 DEBG 363 Reopen extent 34
Sep 22 23:22:21.261 DEBG [2] It's time to notify for 363
Sep 22 23:22:21.261 INFO Completion from [2] id:363 status:true
Sep 22 23:22:21.261 INFO [364/752] Repair commands completed
Sep 22 23:22:21.261 INFO Pop front: ReconcileIO { id: ReconciliationId(364), op: ExtentFlush { repair_id: ReconciliationId(364), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.261 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.261 INFO [0] received reconcile message
Sep 22 23:22:21.261 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(364), op: ExtentFlush { repair_id: ReconciliationId(364), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.261 INFO [0] client ExtentFlush { repair_id: ReconciliationId(364), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.261 INFO [1] received reconcile message
Sep 22 23:22:21.261 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(364), op: ExtentFlush { repair_id: ReconciliationId(364), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.261 INFO [1] client ExtentFlush { repair_id: ReconciliationId(364), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.261 INFO [2] received reconcile message
Sep 22 23:22:21.261 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(364), op: ExtentFlush { repair_id: ReconciliationId(364), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.261 INFO [2] client ExtentFlush { repair_id: ReconciliationId(364), extent_id: 83, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.262 DEBG 364 Flush extent 83 with f:2 g:2
Sep 22 23:22:21.262 DEBG Flush just extent 83 with f:2 and g:2
Sep 22 23:22:21.262 DEBG [1] It's time to notify for 364
Sep 22 23:22:21.262 INFO Completion from [1] id:364 status:true
Sep 22 23:22:21.262 INFO [365/752] Repair commands completed
Sep 22 23:22:21.262 INFO Pop front: ReconcileIO { id: ReconciliationId(365), op: ExtentClose { repair_id: ReconciliationId(365), extent_id: 83 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.262 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.262 INFO [0] received reconcile message
Sep 22 23:22:21.262 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(365), op: ExtentClose { repair_id: ReconciliationId(365), extent_id: 83 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.262 INFO [0] client ExtentClose { repair_id: ReconciliationId(365), extent_id: 83 }
Sep 22 23:22:21.262 INFO [1] received reconcile message
Sep 22 23:22:21.262 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(365), op: ExtentClose { repair_id: ReconciliationId(365), extent_id: 83 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.262 INFO [1] client ExtentClose { repair_id: ReconciliationId(365), extent_id: 83 }
Sep 22 23:22:21.262 INFO [2] received reconcile message
Sep 22 23:22:21.262 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(365), op: ExtentClose { repair_id: ReconciliationId(365), extent_id: 83 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.262 INFO [2] client ExtentClose { repair_id: ReconciliationId(365), extent_id: 83 }
Sep 22 23:22:21.262 DEBG 365 Close extent 83
Sep 22 23:22:21.262 DEBG 365 Close extent 83
Sep 22 23:22:21.263 DEBG 365 Close extent 83
Sep 22 23:22:21.263 DEBG [2] It's time to notify for 365
Sep 22 23:22:21.263 INFO Completion from [2] id:365 status:true
Sep 22 23:22:21.263 INFO [366/752] Repair commands completed
Sep 22 23:22:21.263 INFO Pop front: ReconcileIO { id: ReconciliationId(366), op: ExtentRepair { repair_id: ReconciliationId(366), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.263 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.263 INFO [0] received reconcile message
Sep 22 23:22:21.263 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(366), op: ExtentRepair { repair_id: ReconciliationId(366), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.263 INFO [0] client ExtentRepair { repair_id: ReconciliationId(366), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.263 INFO [0] Sending repair request ReconciliationId(366)
Sep 22 23:22:21.263 INFO [1] received reconcile message
Sep 22 23:22:21.263 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(366), op: ExtentRepair { repair_id: ReconciliationId(366), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.263 INFO [1] client ExtentRepair { repair_id: ReconciliationId(366), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.263 INFO [1] No action required ReconciliationId(366)
Sep 22 23:22:21.263 INFO [2] received reconcile message
Sep 22 23:22:21.264 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(366), op: ExtentRepair { repair_id: ReconciliationId(366), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.264 INFO [2] client ExtentRepair { repair_id: ReconciliationId(366), extent_id: 83, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.264 INFO [2] No action required ReconciliationId(366)
Sep 22 23:22:21.264 DEBG 366 Repair extent 83 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.264 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/053.copy"
Sep 22 23:22:21.265 INFO [lossy] skipping 1084
Sep 22 23:22:21.265 DEBG Flush     :1084 extent_limit None deps:[JobId(1083), JobId(1082)] res:true f:31 g:1
Sep 22 23:22:21.265 INFO [lossy] sleeping 1 second
Sep 22 23:22:21.328 INFO accepted connection, remote_addr: 127.0.0.1:45336, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.328 TRCE incoming request, uri: /extent/83/files, method: GET, req_id: 6bd567c5-c702-4ac8-92de-920bb4d3e083, remote_addr: 127.0.0.1:45336, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.328 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/83/files, method: GET, req_id: 6bd567c5-c702-4ac8-92de-920bb4d3e083, remote_addr: 127.0.0.1:45336, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.329 INFO eid:83 Found repair files: ["053", "053.db"]
Sep 22 23:22:21.329 TRCE incoming request, uri: /newextent/83/data, method: GET, req_id: 952faa8d-c196-40a0-a71b-c104182df9eb, remote_addr: 127.0.0.1:45336, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.329 INFO request completed, latency_us: 322, response_code: 200, uri: /newextent/83/data, method: GET, req_id: 952faa8d-c196-40a0-a71b-c104182df9eb, remote_addr: 127.0.0.1:45336, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.334 TRCE incoming request, uri: /newextent/83/db, method: GET, req_id: d4e5ea97-9f3f-47c5-88ee-b92477f5fe51, remote_addr: 127.0.0.1:45336, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.334 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/83/db, method: GET, req_id: d4e5ea97-9f3f-47c5-88ee-b92477f5fe51, remote_addr: 127.0.0.1:45336, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.335 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/053.copy" to "/tmp/downstairs-zrMnlo6G/00/000/053.replace"
Sep 22 23:22:21.336 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.336 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/053.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.337 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/053"
Sep 22 23:22:21.337 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/053.db"
Sep 22 23:22:21.337 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.337 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/053.replace" to "/tmp/downstairs-zrMnlo6G/00/000/053.completed"
Sep 22 23:22:21.337 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.337 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.337 DEBG [0] It's time to notify for 366
Sep 22 23:22:21.337 INFO Completion from [0] id:366 status:true
Sep 22 23:22:21.337 INFO [367/752] Repair commands completed
Sep 22 23:22:21.337 INFO Pop front: ReconcileIO { id: ReconciliationId(367), op: ExtentReopen { repair_id: ReconciliationId(367), extent_id: 83 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.337 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.337 INFO [0] received reconcile message
Sep 22 23:22:21.337 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(367), op: ExtentReopen { repair_id: ReconciliationId(367), extent_id: 83 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.337 INFO [0] client ExtentReopen { repair_id: ReconciliationId(367), extent_id: 83 }
Sep 22 23:22:21.337 INFO [1] received reconcile message
Sep 22 23:22:21.337 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(367), op: ExtentReopen { repair_id: ReconciliationId(367), extent_id: 83 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.337 INFO [1] client ExtentReopen { repair_id: ReconciliationId(367), extent_id: 83 }
Sep 22 23:22:21.337 INFO [2] received reconcile message
Sep 22 23:22:21.337 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(367), op: ExtentReopen { repair_id: ReconciliationId(367), extent_id: 83 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.337 INFO [2] client ExtentReopen { repair_id: ReconciliationId(367), extent_id: 83 }
Sep 22 23:22:21.338 DEBG 367 Reopen extent 83
Sep 22 23:22:21.338 DEBG 367 Reopen extent 83
Sep 22 23:22:21.339 DEBG 367 Reopen extent 83
Sep 22 23:22:21.339 DEBG [2] It's time to notify for 367
Sep 22 23:22:21.339 INFO Completion from [2] id:367 status:true
Sep 22 23:22:21.339 INFO [368/752] Repair commands completed
Sep 22 23:22:21.339 INFO Pop front: ReconcileIO { id: ReconciliationId(368), op: ExtentFlush { repair_id: ReconciliationId(368), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.339 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.339 INFO [0] received reconcile message
Sep 22 23:22:21.339 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(368), op: ExtentFlush { repair_id: ReconciliationId(368), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.339 INFO [0] client ExtentFlush { repair_id: ReconciliationId(368), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.339 INFO [1] received reconcile message
Sep 22 23:22:21.339 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(368), op: ExtentFlush { repair_id: ReconciliationId(368), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.339 INFO [1] client ExtentFlush { repair_id: ReconciliationId(368), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.340 INFO [2] received reconcile message
Sep 22 23:22:21.340 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(368), op: ExtentFlush { repair_id: ReconciliationId(368), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.340 INFO [2] client ExtentFlush { repair_id: ReconciliationId(368), extent_id: 90, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.340 DEBG 368 Flush extent 90 with f:2 g:2
Sep 22 23:22:21.340 DEBG Flush just extent 90 with f:2 and g:2
Sep 22 23:22:21.340 DEBG [1] It's time to notify for 368
Sep 22 23:22:21.340 INFO Completion from [1] id:368 status:true
Sep 22 23:22:21.340 INFO [369/752] Repair commands completed
Sep 22 23:22:21.340 INFO Pop front: ReconcileIO { id: ReconciliationId(369), op: ExtentClose { repair_id: ReconciliationId(369), extent_id: 90 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.340 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.340 INFO [0] received reconcile message
Sep 22 23:22:21.340 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(369), op: ExtentClose { repair_id: ReconciliationId(369), extent_id: 90 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.340 INFO [0] client ExtentClose { repair_id: ReconciliationId(369), extent_id: 90 }
Sep 22 23:22:21.340 INFO [1] received reconcile message
Sep 22 23:22:21.340 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(369), op: ExtentClose { repair_id: ReconciliationId(369), extent_id: 90 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.340 INFO [1] client ExtentClose { repair_id: ReconciliationId(369), extent_id: 90 }
Sep 22 23:22:21.340 INFO [2] received reconcile message
Sep 22 23:22:21.340 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(369), op: ExtentClose { repair_id: ReconciliationId(369), extent_id: 90 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.340 INFO [2] client ExtentClose { repair_id: ReconciliationId(369), extent_id: 90 }
Sep 22 23:22:21.340 DEBG 369 Close extent 90
Sep 22 23:22:21.341 DEBG 369 Close extent 90
Sep 22 23:22:21.341 DEBG 369 Close extent 90
Sep 22 23:22:21.341 DEBG [2] It's time to notify for 369
Sep 22 23:22:21.341 INFO Completion from [2] id:369 status:true
Sep 22 23:22:21.341 INFO [370/752] Repair commands completed
Sep 22 23:22:21.341 INFO Pop front: ReconcileIO { id: ReconciliationId(370), op: ExtentRepair { repair_id: ReconciliationId(370), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.341 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.341 INFO [0] received reconcile message
Sep 22 23:22:21.341 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(370), op: ExtentRepair { repair_id: ReconciliationId(370), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.341 INFO [0] client ExtentRepair { repair_id: ReconciliationId(370), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.341 INFO [0] Sending repair request ReconciliationId(370)
Sep 22 23:22:21.342 INFO [1] received reconcile message
Sep 22 23:22:21.342 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(370), op: ExtentRepair { repair_id: ReconciliationId(370), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.342 INFO [1] client ExtentRepair { repair_id: ReconciliationId(370), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.342 INFO [1] No action required ReconciliationId(370)
Sep 22 23:22:21.342 INFO [2] received reconcile message
Sep 22 23:22:21.342 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(370), op: ExtentRepair { repair_id: ReconciliationId(370), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.342 INFO [2] client ExtentRepair { repair_id: ReconciliationId(370), extent_id: 90, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.342 INFO [2] No action required ReconciliationId(370)
Sep 22 23:22:21.342 DEBG 370 Repair extent 90 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.342 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/05A.copy"
Sep 22 23:22:21.407 INFO accepted connection, remote_addr: 127.0.0.1:39719, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.407 TRCE incoming request, uri: /extent/90/files, method: GET, req_id: b6cffec7-a064-4229-a79f-0a6741a1bbc8, remote_addr: 127.0.0.1:39719, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.407 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/90/files, method: GET, req_id: b6cffec7-a064-4229-a79f-0a6741a1bbc8, remote_addr: 127.0.0.1:39719, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.408 INFO eid:90 Found repair files: ["05A", "05A.db"]
Sep 22 23:22:21.408 TRCE incoming request, uri: /newextent/90/data, method: GET, req_id: 257516e6-db76-479d-8b49-cf5b481ff065, remote_addr: 127.0.0.1:39719, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.408 INFO request completed, latency_us: 323, response_code: 200, uri: /newextent/90/data, method: GET, req_id: 257516e6-db76-479d-8b49-cf5b481ff065, remote_addr: 127.0.0.1:39719, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.413 TRCE incoming request, uri: /newextent/90/db, method: GET, req_id: 3dc928e5-3c28-4498-8fb7-88853fe93e45, remote_addr: 127.0.0.1:39719, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.413 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/90/db, method: GET, req_id: 3dc928e5-3c28-4498-8fb7-88853fe93e45, remote_addr: 127.0.0.1:39719, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.414 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/05A.copy" to "/tmp/downstairs-zrMnlo6G/00/000/05A.replace"
Sep 22 23:22:21.414 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.415 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/05A.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.415 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05A"
Sep 22 23:22:21.416 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05A.db"
Sep 22 23:22:21.416 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.416 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/05A.replace" to "/tmp/downstairs-zrMnlo6G/00/000/05A.completed"
Sep 22 23:22:21.416 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.416 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.416 DEBG [0] It's time to notify for 370
Sep 22 23:22:21.416 INFO Completion from [0] id:370 status:true
Sep 22 23:22:21.416 INFO [371/752] Repair commands completed
Sep 22 23:22:21.416 INFO Pop front: ReconcileIO { id: ReconciliationId(371), op: ExtentReopen { repair_id: ReconciliationId(371), extent_id: 90 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.416 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.416 INFO [0] received reconcile message
Sep 22 23:22:21.416 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(371), op: ExtentReopen { repair_id: ReconciliationId(371), extent_id: 90 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.416 INFO [0] client ExtentReopen { repair_id: ReconciliationId(371), extent_id: 90 }
Sep 22 23:22:21.416 INFO [1] received reconcile message
Sep 22 23:22:21.416 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(371), op: ExtentReopen { repair_id: ReconciliationId(371), extent_id: 90 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.416 INFO [1] client ExtentReopen { repair_id: ReconciliationId(371), extent_id: 90 }
Sep 22 23:22:21.416 INFO [2] received reconcile message
Sep 22 23:22:21.416 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(371), op: ExtentReopen { repair_id: ReconciliationId(371), extent_id: 90 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.416 INFO [2] client ExtentReopen { repair_id: ReconciliationId(371), extent_id: 90 }
Sep 22 23:22:21.416 DEBG 371 Reopen extent 90
Sep 22 23:22:21.417 DEBG 371 Reopen extent 90
Sep 22 23:22:21.418 DEBG 371 Reopen extent 90
Sep 22 23:22:21.418 DEBG [2] It's time to notify for 371
Sep 22 23:22:21.418 INFO Completion from [2] id:371 status:true
Sep 22 23:22:21.418 INFO [372/752] Repair commands completed
Sep 22 23:22:21.418 INFO Pop front: ReconcileIO { id: ReconciliationId(372), op: ExtentFlush { repair_id: ReconciliationId(372), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.418 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.418 INFO [0] received reconcile message
Sep 22 23:22:21.418 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(372), op: ExtentFlush { repair_id: ReconciliationId(372), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.418 INFO [0] client ExtentFlush { repair_id: ReconciliationId(372), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.418 INFO [1] received reconcile message
Sep 22 23:22:21.418 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(372), op: ExtentFlush { repair_id: ReconciliationId(372), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.418 INFO [1] client ExtentFlush { repair_id: ReconciliationId(372), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.418 INFO [2] received reconcile message
Sep 22 23:22:21.418 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(372), op: ExtentFlush { repair_id: ReconciliationId(372), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.418 INFO [2] client ExtentFlush { repair_id: ReconciliationId(372), extent_id: 66, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.419 DEBG 372 Flush extent 66 with f:2 g:2
Sep 22 23:22:21.419 DEBG Flush just extent 66 with f:2 and g:2
Sep 22 23:22:21.419 DEBG [1] It's time to notify for 372
Sep 22 23:22:21.419 INFO Completion from [1] id:372 status:true
Sep 22 23:22:21.419 INFO [373/752] Repair commands completed
Sep 22 23:22:21.419 INFO Pop front: ReconcileIO { id: ReconciliationId(373), op: ExtentClose { repair_id: ReconciliationId(373), extent_id: 66 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.419 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.419 INFO [0] received reconcile message
Sep 22 23:22:21.419 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(373), op: ExtentClose { repair_id: ReconciliationId(373), extent_id: 66 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.419 INFO [0] client ExtentClose { repair_id: ReconciliationId(373), extent_id: 66 }
Sep 22 23:22:21.419 INFO [1] received reconcile message
Sep 22 23:22:21.419 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(373), op: ExtentClose { repair_id: ReconciliationId(373), extent_id: 66 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.419 INFO [1] client ExtentClose { repair_id: ReconciliationId(373), extent_id: 66 }
Sep 22 23:22:21.419 INFO [2] received reconcile message
Sep 22 23:22:21.419 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(373), op: ExtentClose { repair_id: ReconciliationId(373), extent_id: 66 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.419 INFO [2] client ExtentClose { repair_id: ReconciliationId(373), extent_id: 66 }
Sep 22 23:22:21.419 DEBG 373 Close extent 66
Sep 22 23:22:21.419 DEBG 373 Close extent 66
Sep 22 23:22:21.420 DEBG 373 Close extent 66
Sep 22 23:22:21.420 DEBG [2] It's time to notify for 373
Sep 22 23:22:21.420 INFO Completion from [2] id:373 status:true
Sep 22 23:22:21.420 INFO [374/752] Repair commands completed
Sep 22 23:22:21.420 INFO Pop front: ReconcileIO { id: ReconciliationId(374), op: ExtentRepair { repair_id: ReconciliationId(374), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.420 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.420 INFO [0] received reconcile message
Sep 22 23:22:21.420 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(374), op: ExtentRepair { repair_id: ReconciliationId(374), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.420 INFO [0] client ExtentRepair { repair_id: ReconciliationId(374), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.420 INFO [0] Sending repair request ReconciliationId(374)
Sep 22 23:22:21.420 INFO [1] received reconcile message
Sep 22 23:22:21.420 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(374), op: ExtentRepair { repair_id: ReconciliationId(374), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.420 INFO [1] client ExtentRepair { repair_id: ReconciliationId(374), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.420 INFO [1] No action required ReconciliationId(374)
Sep 22 23:22:21.421 INFO [2] received reconcile message
Sep 22 23:22:21.421 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(374), op: ExtentRepair { repair_id: ReconciliationId(374), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.421 INFO [2] client ExtentRepair { repair_id: ReconciliationId(374), extent_id: 66, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.421 INFO [2] No action required ReconciliationId(374)
Sep 22 23:22:21.421 DEBG 374 Repair extent 66 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.421 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/042.copy"
Sep 22 23:22:21.484 INFO accepted connection, remote_addr: 127.0.0.1:60013, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.485 TRCE incoming request, uri: /extent/66/files, method: GET, req_id: ba3e1e6d-a638-494f-8f68-929d0d0b6c4b, remote_addr: 127.0.0.1:60013, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.485 INFO request completed, latency_us: 208, response_code: 200, uri: /extent/66/files, method: GET, req_id: ba3e1e6d-a638-494f-8f68-929d0d0b6c4b, remote_addr: 127.0.0.1:60013, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.485 INFO eid:66 Found repair files: ["042", "042.db"]
Sep 22 23:22:21.485 TRCE incoming request, uri: /newextent/66/data, method: GET, req_id: 631ac310-2844-403e-bf4c-d141a93d8a8f, remote_addr: 127.0.0.1:60013, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.486 INFO request completed, latency_us: 317, response_code: 200, uri: /newextent/66/data, method: GET, req_id: 631ac310-2844-403e-bf4c-d141a93d8a8f, remote_addr: 127.0.0.1:60013, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.490 TRCE incoming request, uri: /newextent/66/db, method: GET, req_id: e36ffdaa-88ba-4641-be62-75003b4668b8, remote_addr: 127.0.0.1:60013, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.491 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/66/db, method: GET, req_id: e36ffdaa-88ba-4641-be62-75003b4668b8, remote_addr: 127.0.0.1:60013, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.492 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/042.copy" to "/tmp/downstairs-zrMnlo6G/00/000/042.replace"
Sep 22 23:22:21.492 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.493 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/042.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.493 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/042"
Sep 22 23:22:21.493 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/042.db"
Sep 22 23:22:21.493 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.493 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/042.replace" to "/tmp/downstairs-zrMnlo6G/00/000/042.completed"
Sep 22 23:22:21.493 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.493 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.493 DEBG [0] It's time to notify for 374
Sep 22 23:22:21.493 INFO Completion from [0] id:374 status:true
Sep 22 23:22:21.494 INFO [375/752] Repair commands completed
Sep 22 23:22:21.494 INFO Pop front: ReconcileIO { id: ReconciliationId(375), op: ExtentReopen { repair_id: ReconciliationId(375), extent_id: 66 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.494 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.494 INFO [0] received reconcile message
Sep 22 23:22:21.494 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(375), op: ExtentReopen { repair_id: ReconciliationId(375), extent_id: 66 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.494 INFO [0] client ExtentReopen { repair_id: ReconciliationId(375), extent_id: 66 }
Sep 22 23:22:21.494 INFO [1] received reconcile message
Sep 22 23:22:21.494 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(375), op: ExtentReopen { repair_id: ReconciliationId(375), extent_id: 66 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.494 INFO [1] client ExtentReopen { repair_id: ReconciliationId(375), extent_id: 66 }
Sep 22 23:22:21.494 INFO [2] received reconcile message
Sep 22 23:22:21.494 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(375), op: ExtentReopen { repair_id: ReconciliationId(375), extent_id: 66 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.494 INFO [2] client ExtentReopen { repair_id: ReconciliationId(375), extent_id: 66 }
Sep 22 23:22:21.494 DEBG 375 Reopen extent 66
Sep 22 23:22:21.494 DEBG 375 Reopen extent 66
Sep 22 23:22:21.495 DEBG 375 Reopen extent 66
Sep 22 23:22:21.496 DEBG [2] It's time to notify for 375
Sep 22 23:22:21.496 INFO Completion from [2] id:375 status:true
Sep 22 23:22:21.496 INFO [376/752] Repair commands completed
Sep 22 23:22:21.496 INFO Pop front: ReconcileIO { id: ReconciliationId(376), op: ExtentFlush { repair_id: ReconciliationId(376), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.496 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.496 INFO [0] received reconcile message
Sep 22 23:22:21.496 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(376), op: ExtentFlush { repair_id: ReconciliationId(376), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.496 INFO [0] client ExtentFlush { repair_id: ReconciliationId(376), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.496 INFO [1] received reconcile message
Sep 22 23:22:21.496 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(376), op: ExtentFlush { repair_id: ReconciliationId(376), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.496 INFO [1] client ExtentFlush { repair_id: ReconciliationId(376), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.496 INFO [2] received reconcile message
Sep 22 23:22:21.496 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(376), op: ExtentFlush { repair_id: ReconciliationId(376), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.496 INFO [2] client ExtentFlush { repair_id: ReconciliationId(376), extent_id: 87, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.496 DEBG 376 Flush extent 87 with f:2 g:2
Sep 22 23:22:21.496 DEBG Flush just extent 87 with f:2 and g:2
Sep 22 23:22:21.496 DEBG [1] It's time to notify for 376
Sep 22 23:22:21.496 INFO Completion from [1] id:376 status:true
Sep 22 23:22:21.496 INFO [377/752] Repair commands completed
Sep 22 23:22:21.496 INFO Pop front: ReconcileIO { id: ReconciliationId(377), op: ExtentClose { repair_id: ReconciliationId(377), extent_id: 87 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.496 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.496 INFO [0] received reconcile message
Sep 22 23:22:21.496 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(377), op: ExtentClose { repair_id: ReconciliationId(377), extent_id: 87 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.496 INFO [0] client ExtentClose { repair_id: ReconciliationId(377), extent_id: 87 }
Sep 22 23:22:21.496 INFO [1] received reconcile message
Sep 22 23:22:21.496 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(377), op: ExtentClose { repair_id: ReconciliationId(377), extent_id: 87 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.496 INFO [1] client ExtentClose { repair_id: ReconciliationId(377), extent_id: 87 }
Sep 22 23:22:21.497 INFO [2] received reconcile message
Sep 22 23:22:21.497 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(377), op: ExtentClose { repair_id: ReconciliationId(377), extent_id: 87 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.497 INFO [2] client ExtentClose { repair_id: ReconciliationId(377), extent_id: 87 }
Sep 22 23:22:21.497 DEBG 377 Close extent 87
Sep 22 23:22:21.497 DEBG 377 Close extent 87
Sep 22 23:22:21.497 DEBG 377 Close extent 87
Sep 22 23:22:21.498 DEBG [2] It's time to notify for 377
Sep 22 23:22:21.498 INFO Completion from [2] id:377 status:true
Sep 22 23:22:21.498 INFO [378/752] Repair commands completed
Sep 22 23:22:21.498 INFO Pop front: ReconcileIO { id: ReconciliationId(378), op: ExtentRepair { repair_id: ReconciliationId(378), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.498 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.498 INFO [0] received reconcile message
Sep 22 23:22:21.498 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(378), op: ExtentRepair { repair_id: ReconciliationId(378), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.498 INFO [0] client ExtentRepair { repair_id: ReconciliationId(378), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.498 INFO [0] Sending repair request ReconciliationId(378)
Sep 22 23:22:21.498 INFO [1] received reconcile message
Sep 22 23:22:21.498 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(378), op: ExtentRepair { repair_id: ReconciliationId(378), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.498 INFO [1] client ExtentRepair { repair_id: ReconciliationId(378), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.498 INFO [1] No action required ReconciliationId(378)
Sep 22 23:22:21.498 INFO [2] received reconcile message
Sep 22 23:22:21.498 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(378), op: ExtentRepair { repair_id: ReconciliationId(378), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.498 INFO [2] client ExtentRepair { repair_id: ReconciliationId(378), extent_id: 87, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.498 INFO [2] No action required ReconciliationId(378)
Sep 22 23:22:21.498 DEBG 378 Repair extent 87 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.498 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/057.copy"
Sep 22 23:22:21.563 INFO accepted connection, remote_addr: 127.0.0.1:47158, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.563 TRCE incoming request, uri: /extent/87/files, method: GET, req_id: 798788a8-e856-46a4-ac2d-625351a34e85, remote_addr: 127.0.0.1:47158, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.564 INFO request completed, latency_us: 194, response_code: 200, uri: /extent/87/files, method: GET, req_id: 798788a8-e856-46a4-ac2d-625351a34e85, remote_addr: 127.0.0.1:47158, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.564 INFO eid:87 Found repair files: ["057", "057.db"]
Sep 22 23:22:21.564 TRCE incoming request, uri: /newextent/87/data, method: GET, req_id: 8f8ed8c8-a65e-413a-8d94-d772584e4e4b, remote_addr: 127.0.0.1:47158, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.565 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/87/data, method: GET, req_id: 8f8ed8c8-a65e-413a-8d94-d772584e4e4b, remote_addr: 127.0.0.1:47158, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.569 TRCE incoming request, uri: /newextent/87/db, method: GET, req_id: 17351486-c099-4e22-a231-27f8e59fa155, remote_addr: 127.0.0.1:47158, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.569 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/87/db, method: GET, req_id: 17351486-c099-4e22-a231-27f8e59fa155, remote_addr: 127.0.0.1:47158, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.571 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/057.copy" to "/tmp/downstairs-zrMnlo6G/00/000/057.replace"
Sep 22 23:22:21.571 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.571 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/057.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.572 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/057"
Sep 22 23:22:21.572 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/057.db"
Sep 22 23:22:21.572 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.572 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/057.replace" to "/tmp/downstairs-zrMnlo6G/00/000/057.completed"
Sep 22 23:22:21.572 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.572 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.572 DEBG [0] It's time to notify for 378
Sep 22 23:22:21.572 INFO Completion from [0] id:378 status:true
Sep 22 23:22:21.572 INFO [379/752] Repair commands completed
Sep 22 23:22:21.572 INFO Pop front: ReconcileIO { id: ReconciliationId(379), op: ExtentReopen { repair_id: ReconciliationId(379), extent_id: 87 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.572 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.572 INFO [0] received reconcile message
Sep 22 23:22:21.572 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(379), op: ExtentReopen { repair_id: ReconciliationId(379), extent_id: 87 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.572 INFO [0] client ExtentReopen { repair_id: ReconciliationId(379), extent_id: 87 }
Sep 22 23:22:21.572 INFO [1] received reconcile message
Sep 22 23:22:21.572 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(379), op: ExtentReopen { repair_id: ReconciliationId(379), extent_id: 87 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.572 INFO [1] client ExtentReopen { repair_id: ReconciliationId(379), extent_id: 87 }
Sep 22 23:22:21.572 INFO [2] received reconcile message
Sep 22 23:22:21.572 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(379), op: ExtentReopen { repair_id: ReconciliationId(379), extent_id: 87 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.572 INFO [2] client ExtentReopen { repair_id: ReconciliationId(379), extent_id: 87 }
Sep 22 23:22:21.573 DEBG 379 Reopen extent 87
Sep 22 23:22:21.573 DEBG 379 Reopen extent 87
Sep 22 23:22:21.574 DEBG 379 Reopen extent 87
Sep 22 23:22:21.574 DEBG [2] It's time to notify for 379
Sep 22 23:22:21.574 INFO Completion from [2] id:379 status:true
Sep 22 23:22:21.574 INFO [380/752] Repair commands completed
Sep 22 23:22:21.574 INFO Pop front: ReconcileIO { id: ReconciliationId(380), op: ExtentFlush { repair_id: ReconciliationId(380), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.574 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.574 INFO [0] received reconcile message
Sep 22 23:22:21.574 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(380), op: ExtentFlush { repair_id: ReconciliationId(380), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.574 INFO [0] client ExtentFlush { repair_id: ReconciliationId(380), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.574 INFO [1] received reconcile message
Sep 22 23:22:21.575 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(380), op: ExtentFlush { repair_id: ReconciliationId(380), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.575 INFO [1] client ExtentFlush { repair_id: ReconciliationId(380), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.575 INFO [2] received reconcile message
Sep 22 23:22:21.575 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(380), op: ExtentFlush { repair_id: ReconciliationId(380), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.575 INFO [2] client ExtentFlush { repair_id: ReconciliationId(380), extent_id: 139, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.575 DEBG 380 Flush extent 139 with f:2 g:2
Sep 22 23:22:21.575 DEBG Flush just extent 139 with f:2 and g:2
Sep 22 23:22:21.575 DEBG [1] It's time to notify for 380
Sep 22 23:22:21.575 INFO Completion from [1] id:380 status:true
Sep 22 23:22:21.575 INFO [381/752] Repair commands completed
Sep 22 23:22:21.575 INFO Pop front: ReconcileIO { id: ReconciliationId(381), op: ExtentClose { repair_id: ReconciliationId(381), extent_id: 139 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.575 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.575 INFO [0] received reconcile message
Sep 22 23:22:21.575 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(381), op: ExtentClose { repair_id: ReconciliationId(381), extent_id: 139 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.575 INFO [0] client ExtentClose { repair_id: ReconciliationId(381), extent_id: 139 }
Sep 22 23:22:21.575 INFO [1] received reconcile message
Sep 22 23:22:21.575 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(381), op: ExtentClose { repair_id: ReconciliationId(381), extent_id: 139 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.575 INFO [1] client ExtentClose { repair_id: ReconciliationId(381), extent_id: 139 }
Sep 22 23:22:21.575 INFO [2] received reconcile message
Sep 22 23:22:21.575 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(381), op: ExtentClose { repair_id: ReconciliationId(381), extent_id: 139 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.575 INFO [2] client ExtentClose { repair_id: ReconciliationId(381), extent_id: 139 }
Sep 22 23:22:21.575 DEBG 381 Close extent 139
Sep 22 23:22:21.576 DEBG 381 Close extent 139
Sep 22 23:22:21.576 DEBG 381 Close extent 139
Sep 22 23:22:21.576 DEBG [2] It's time to notify for 381
Sep 22 23:22:21.576 INFO Completion from [2] id:381 status:true
Sep 22 23:22:21.576 INFO [382/752] Repair commands completed
Sep 22 23:22:21.576 INFO Pop front: ReconcileIO { id: ReconciliationId(382), op: ExtentRepair { repair_id: ReconciliationId(382), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.576 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.576 INFO [0] received reconcile message
Sep 22 23:22:21.576 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(382), op: ExtentRepair { repair_id: ReconciliationId(382), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.576 INFO [0] client ExtentRepair { repair_id: ReconciliationId(382), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.576 INFO [0] Sending repair request ReconciliationId(382)
Sep 22 23:22:21.577 INFO [1] received reconcile message
Sep 22 23:22:21.577 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(382), op: ExtentRepair { repair_id: ReconciliationId(382), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.577 INFO [1] client ExtentRepair { repair_id: ReconciliationId(382), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.577 INFO [1] No action required ReconciliationId(382)
Sep 22 23:22:21.577 INFO [2] received reconcile message
Sep 22 23:22:21.577 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(382), op: ExtentRepair { repair_id: ReconciliationId(382), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.577 INFO [2] client ExtentRepair { repair_id: ReconciliationId(382), extent_id: 139, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.577 INFO [2] No action required ReconciliationId(382)
Sep 22 23:22:21.577 DEBG 382 Repair extent 139 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.577 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/08B.copy"
Sep 22 23:22:21.641 INFO accepted connection, remote_addr: 127.0.0.1:35693, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.641 DEBG [1] Read AckReady 1083, : downstairs
Sep 22 23:22:21.641 TRCE incoming request, uri: /extent/139/files, method: GET, req_id: 5845da20-bc49-4ff0-b9d8-91c777237045, remote_addr: 127.0.0.1:35693, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.641 INFO request completed, latency_us: 196, response_code: 200, uri: /extent/139/files, method: GET, req_id: 5845da20-bc49-4ff0-b9d8-91c777237045, remote_addr: 127.0.0.1:35693, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.642 INFO eid:139 Found repair files: ["08B", "08B.db"]
Sep 22 23:22:21.642 TRCE incoming request, uri: /newextent/139/data, method: GET, req_id: 824f179f-3cf6-4070-bcb5-5322ad003d5d, remote_addr: 127.0.0.1:35693, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.642 DEBG up_ds_listen was notified
Sep 22 23:22:21.642 DEBG up_ds_listen process 1083
Sep 22 23:22:21.642 DEBG [A] ack job 1083:84, : downstairs
Sep 22 23:22:21.642 INFO request completed, latency_us: 324, response_code: 200, uri: /newextent/139/data, method: GET, req_id: 824f179f-3cf6-4070-bcb5-5322ad003d5d, remote_addr: 127.0.0.1:35693, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.647 TRCE incoming request, uri: /newextent/139/db, method: GET, req_id: 331fb13c-587f-4504-a055-5896edbe3bc5, remote_addr: 127.0.0.1:35693, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.647 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/139/db, method: GET, req_id: 331fb13c-587f-4504-a055-5896edbe3bc5, remote_addr: 127.0.0.1:35693, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.648 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/08B.copy" to "/tmp/downstairs-zrMnlo6G/00/000/08B.replace"
Sep 22 23:22:21.648 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.649 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/08B.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.649 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08B"
Sep 22 23:22:21.650 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08B.db"
Sep 22 23:22:21.650 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.650 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/08B.replace" to "/tmp/downstairs-zrMnlo6G/00/000/08B.completed"
Sep 22 23:22:21.650 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.650 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.650 DEBG [0] It's time to notify for 382
Sep 22 23:22:21.650 INFO Completion from [0] id:382 status:true
Sep 22 23:22:21.650 INFO [383/752] Repair commands completed
Sep 22 23:22:21.650 INFO Pop front: ReconcileIO { id: ReconciliationId(383), op: ExtentReopen { repair_id: ReconciliationId(383), extent_id: 139 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.650 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.650 INFO [0] received reconcile message
Sep 22 23:22:21.650 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(383), op: ExtentReopen { repair_id: ReconciliationId(383), extent_id: 139 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.650 INFO [0] client ExtentReopen { repair_id: ReconciliationId(383), extent_id: 139 }
Sep 22 23:22:21.650 INFO [1] received reconcile message
Sep 22 23:22:21.650 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(383), op: ExtentReopen { repair_id: ReconciliationId(383), extent_id: 139 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.650 INFO [1] client ExtentReopen { repair_id: ReconciliationId(383), extent_id: 139 }
Sep 22 23:22:21.650 INFO [2] received reconcile message
Sep 22 23:22:21.650 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(383), op: ExtentReopen { repair_id: ReconciliationId(383), extent_id: 139 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.650 INFO [2] client ExtentReopen { repair_id: ReconciliationId(383), extent_id: 139 }
Sep 22 23:22:21.650 DEBG 383 Reopen extent 139
Sep 22 23:22:21.651 DEBG 383 Reopen extent 139
Sep 22 23:22:21.651 DEBG 383 Reopen extent 139
Sep 22 23:22:21.652 DEBG [2] It's time to notify for 383
Sep 22 23:22:21.652 INFO Completion from [2] id:383 status:true
Sep 22 23:22:21.652 INFO [384/752] Repair commands completed
Sep 22 23:22:21.652 INFO Pop front: ReconcileIO { id: ReconciliationId(384), op: ExtentFlush { repair_id: ReconciliationId(384), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.652 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.652 INFO [0] received reconcile message
Sep 22 23:22:21.652 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(384), op: ExtentFlush { repair_id: ReconciliationId(384), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.652 INFO [0] client ExtentFlush { repair_id: ReconciliationId(384), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.652 INFO [1] received reconcile message
Sep 22 23:22:21.652 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(384), op: ExtentFlush { repair_id: ReconciliationId(384), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.652 INFO [1] client ExtentFlush { repair_id: ReconciliationId(384), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.652 INFO [2] received reconcile message
Sep 22 23:22:21.652 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(384), op: ExtentFlush { repair_id: ReconciliationId(384), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.652 INFO [2] client ExtentFlush { repair_id: ReconciliationId(384), extent_id: 26, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.653 DEBG 384 Flush extent 26 with f:2 g:2
Sep 22 23:22:21.653 DEBG Flush just extent 26 with f:2 and g:2
Sep 22 23:22:21.653 DEBG [1] It's time to notify for 384
Sep 22 23:22:21.653 INFO Completion from [1] id:384 status:true
Sep 22 23:22:21.653 INFO [385/752] Repair commands completed
Sep 22 23:22:21.653 INFO Pop front: ReconcileIO { id: ReconciliationId(385), op: ExtentClose { repair_id: ReconciliationId(385), extent_id: 26 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.653 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.653 INFO [0] received reconcile message
Sep 22 23:22:21.653 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(385), op: ExtentClose { repair_id: ReconciliationId(385), extent_id: 26 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.653 INFO [0] client ExtentClose { repair_id: ReconciliationId(385), extent_id: 26 }
Sep 22 23:22:21.653 INFO [1] received reconcile message
Sep 22 23:22:21.653 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(385), op: ExtentClose { repair_id: ReconciliationId(385), extent_id: 26 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.653 INFO [1] client ExtentClose { repair_id: ReconciliationId(385), extent_id: 26 }
Sep 22 23:22:21.653 INFO [2] received reconcile message
Sep 22 23:22:21.653 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(385), op: ExtentClose { repair_id: ReconciliationId(385), extent_id: 26 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.653 INFO [2] client ExtentClose { repair_id: ReconciliationId(385), extent_id: 26 }
Sep 22 23:22:21.653 DEBG 385 Close extent 26
Sep 22 23:22:21.653 DEBG 385 Close extent 26
Sep 22 23:22:21.654 DEBG 385 Close extent 26
Sep 22 23:22:21.654 DEBG [2] It's time to notify for 385
Sep 22 23:22:21.654 INFO Completion from [2] id:385 status:true
Sep 22 23:22:21.654 INFO [386/752] Repair commands completed
Sep 22 23:22:21.654 INFO Pop front: ReconcileIO { id: ReconciliationId(386), op: ExtentRepair { repair_id: ReconciliationId(386), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.654 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.654 INFO [0] received reconcile message
Sep 22 23:22:21.654 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(386), op: ExtentRepair { repair_id: ReconciliationId(386), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.654 INFO [0] client ExtentRepair { repair_id: ReconciliationId(386), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.654 INFO [0] Sending repair request ReconciliationId(386)
Sep 22 23:22:21.654 INFO [1] received reconcile message
Sep 22 23:22:21.654 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(386), op: ExtentRepair { repair_id: ReconciliationId(386), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.654 INFO [1] client ExtentRepair { repair_id: ReconciliationId(386), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.654 INFO [1] No action required ReconciliationId(386)
Sep 22 23:22:21.654 INFO [2] received reconcile message
Sep 22 23:22:21.654 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(386), op: ExtentRepair { repair_id: ReconciliationId(386), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.654 INFO [2] client ExtentRepair { repair_id: ReconciliationId(386), extent_id: 26, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.654 INFO [2] No action required ReconciliationId(386)
Sep 22 23:22:21.655 DEBG 386 Repair extent 26 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.655 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/01A.copy"
Sep 22 23:22:21.695 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:21.702 DEBG Read      :1083 deps:[JobId(1082)] res:true
Sep 22 23:22:21.717 INFO accepted connection, remote_addr: 127.0.0.1:47355, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.717 TRCE incoming request, uri: /extent/26/files, method: GET, req_id: e7f8ec6c-c012-4ced-9835-4563c69666fc, remote_addr: 127.0.0.1:47355, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.718 INFO request completed, latency_us: 238, response_code: 200, uri: /extent/26/files, method: GET, req_id: e7f8ec6c-c012-4ced-9835-4563c69666fc, remote_addr: 127.0.0.1:47355, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.718 INFO eid:26 Found repair files: ["01A", "01A.db"]
Sep 22 23:22:21.718 TRCE incoming request, uri: /newextent/26/data, method: GET, req_id: 3d9d7983-f6f3-451c-a32a-433c198e06b5, remote_addr: 127.0.0.1:47355, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.719 INFO request completed, latency_us: 333, response_code: 200, uri: /newextent/26/data, method: GET, req_id: 3d9d7983-f6f3-451c-a32a-433c198e06b5, remote_addr: 127.0.0.1:47355, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.723 TRCE incoming request, uri: /newextent/26/db, method: GET, req_id: 537fe346-99aa-4ce6-a2b6-f177755c4d6c, remote_addr: 127.0.0.1:47355, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.723 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/26/db, method: GET, req_id: 537fe346-99aa-4ce6-a2b6-f177755c4d6c, remote_addr: 127.0.0.1:47355, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.724 DEBG IO Read  1085 has deps [JobId(1084)]
Sep 22 23:22:21.725 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/01A.copy" to "/tmp/downstairs-zrMnlo6G/00/000/01A.replace"
Sep 22 23:22:21.725 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.726 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/01A.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.726 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01A"
Sep 22 23:22:21.726 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01A.db"
Sep 22 23:22:21.726 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.726 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/01A.replace" to "/tmp/downstairs-zrMnlo6G/00/000/01A.completed"
Sep 22 23:22:21.726 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.726 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.726 DEBG [0] It's time to notify for 386
Sep 22 23:22:21.726 INFO Completion from [0] id:386 status:true
Sep 22 23:22:21.726 INFO [387/752] Repair commands completed
Sep 22 23:22:21.726 INFO Pop front: ReconcileIO { id: ReconciliationId(387), op: ExtentReopen { repair_id: ReconciliationId(387), extent_id: 26 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.726 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.727 INFO [0] received reconcile message
Sep 22 23:22:21.727 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(387), op: ExtentReopen { repair_id: ReconciliationId(387), extent_id: 26 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.727 INFO [0] client ExtentReopen { repair_id: ReconciliationId(387), extent_id: 26 }
Sep 22 23:22:21.727 INFO [1] received reconcile message
Sep 22 23:22:21.727 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(387), op: ExtentReopen { repair_id: ReconciliationId(387), extent_id: 26 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.727 INFO [1] client ExtentReopen { repair_id: ReconciliationId(387), extent_id: 26 }
Sep 22 23:22:21.727 INFO [2] received reconcile message
Sep 22 23:22:21.727 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(387), op: ExtentReopen { repair_id: ReconciliationId(387), extent_id: 26 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.727 INFO [2] client ExtentReopen { repair_id: ReconciliationId(387), extent_id: 26 }
Sep 22 23:22:21.727 DEBG 387 Reopen extent 26
Sep 22 23:22:21.728 DEBG 387 Reopen extent 26
Sep 22 23:22:21.728 DEBG 387 Reopen extent 26
Sep 22 23:22:21.729 DEBG [2] It's time to notify for 387
Sep 22 23:22:21.729 INFO Completion from [2] id:387 status:true
Sep 22 23:22:21.729 INFO [388/752] Repair commands completed
Sep 22 23:22:21.729 INFO Pop front: ReconcileIO { id: ReconciliationId(388), op: ExtentFlush { repair_id: ReconciliationId(388), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.729 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.729 INFO [0] received reconcile message
Sep 22 23:22:21.729 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(388), op: ExtentFlush { repair_id: ReconciliationId(388), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.729 INFO [0] client ExtentFlush { repair_id: ReconciliationId(388), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.729 INFO [1] received reconcile message
Sep 22 23:22:21.729 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(388), op: ExtentFlush { repair_id: ReconciliationId(388), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.729 INFO [1] client ExtentFlush { repair_id: ReconciliationId(388), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.729 INFO [2] received reconcile message
Sep 22 23:22:21.729 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(388), op: ExtentFlush { repair_id: ReconciliationId(388), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.729 INFO [2] client ExtentFlush { repair_id: ReconciliationId(388), extent_id: 84, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.729 DEBG 388 Flush extent 84 with f:2 g:2
Sep 22 23:22:21.729 DEBG Flush just extent 84 with f:2 and g:2
Sep 22 23:22:21.729 DEBG [1] It's time to notify for 388
Sep 22 23:22:21.729 INFO Completion from [1] id:388 status:true
Sep 22 23:22:21.729 INFO [389/752] Repair commands completed
Sep 22 23:22:21.729 INFO Pop front: ReconcileIO { id: ReconciliationId(389), op: ExtentClose { repair_id: ReconciliationId(389), extent_id: 84 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.729 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.729 INFO [0] received reconcile message
Sep 22 23:22:21.729 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(389), op: ExtentClose { repair_id: ReconciliationId(389), extent_id: 84 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.729 INFO [0] client ExtentClose { repair_id: ReconciliationId(389), extent_id: 84 }
Sep 22 23:22:21.730 INFO [1] received reconcile message
Sep 22 23:22:21.730 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(389), op: ExtentClose { repair_id: ReconciliationId(389), extent_id: 84 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.730 INFO [1] client ExtentClose { repair_id: ReconciliationId(389), extent_id: 84 }
Sep 22 23:22:21.730 INFO [2] received reconcile message
Sep 22 23:22:21.730 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(389), op: ExtentClose { repair_id: ReconciliationId(389), extent_id: 84 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.730 INFO [2] client ExtentClose { repair_id: ReconciliationId(389), extent_id: 84 }
Sep 22 23:22:21.730 DEBG 389 Close extent 84
Sep 22 23:22:21.730 DEBG 389 Close extent 84
Sep 22 23:22:21.730 DEBG 389 Close extent 84
Sep 22 23:22:21.731 DEBG [2] It's time to notify for 389
Sep 22 23:22:21.731 INFO Completion from [2] id:389 status:true
Sep 22 23:22:21.731 INFO [390/752] Repair commands completed
Sep 22 23:22:21.731 INFO Pop front: ReconcileIO { id: ReconciliationId(390), op: ExtentRepair { repair_id: ReconciliationId(390), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.731 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.731 INFO [0] received reconcile message
Sep 22 23:22:21.731 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(390), op: ExtentRepair { repair_id: ReconciliationId(390), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.731 INFO [0] client ExtentRepair { repair_id: ReconciliationId(390), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.731 INFO [0] Sending repair request ReconciliationId(390)
Sep 22 23:22:21.731 INFO [1] received reconcile message
Sep 22 23:22:21.731 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(390), op: ExtentRepair { repair_id: ReconciliationId(390), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.731 INFO [1] client ExtentRepair { repair_id: ReconciliationId(390), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.731 INFO [1] No action required ReconciliationId(390)
Sep 22 23:22:21.731 INFO [2] received reconcile message
Sep 22 23:22:21.731 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(390), op: ExtentRepair { repair_id: ReconciliationId(390), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.731 INFO [2] client ExtentRepair { repair_id: ReconciliationId(390), extent_id: 84, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.731 INFO [2] No action required ReconciliationId(390)
Sep 22 23:22:21.731 DEBG 390 Repair extent 84 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.731 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/054.copy"
Sep 22 23:22:21.737 INFO [lossy] skipping 1084
Sep 22 23:22:21.737 INFO [lossy] skipping 1084
Sep 22 23:22:21.737 DEBG Flush     :1084 extent_limit None deps:[JobId(1083), JobId(1082)] res:true f:31 g:1
Sep 22 23:22:21.743 DEBG Read      :1085 deps:[JobId(1084)] res:true
Sep 22 23:22:21.793 INFO accepted connection, remote_addr: 127.0.0.1:49228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.793 TRCE incoming request, uri: /extent/84/files, method: GET, req_id: 5da15b25-742b-4fee-a6bc-2f22b72ff218, remote_addr: 127.0.0.1:49228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.793 INFO request completed, latency_us: 207, response_code: 200, uri: /extent/84/files, method: GET, req_id: 5da15b25-742b-4fee-a6bc-2f22b72ff218, remote_addr: 127.0.0.1:49228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.794 INFO eid:84 Found repair files: ["054", "054.db"]
Sep 22 23:22:21.794 TRCE incoming request, uri: /newextent/84/data, method: GET, req_id: 138618f5-ba37-4436-9b12-f4c7fb1df2ce, remote_addr: 127.0.0.1:49228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.794 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/84/data, method: GET, req_id: 138618f5-ba37-4436-9b12-f4c7fb1df2ce, remote_addr: 127.0.0.1:49228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.799 TRCE incoming request, uri: /newextent/84/db, method: GET, req_id: 9bde8b18-754a-45f8-843e-b1c49c499d45, remote_addr: 127.0.0.1:49228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.799 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/84/db, method: GET, req_id: 9bde8b18-754a-45f8-843e-b1c49c499d45, remote_addr: 127.0.0.1:49228, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.801 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/054.copy" to "/tmp/downstairs-zrMnlo6G/00/000/054.replace"
Sep 22 23:22:21.801 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.801 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/054.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.802 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/054"
Sep 22 23:22:21.802 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/054.db"
Sep 22 23:22:21.802 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.802 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/054.replace" to "/tmp/downstairs-zrMnlo6G/00/000/054.completed"
Sep 22 23:22:21.802 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.802 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.802 DEBG [0] It's time to notify for 390
Sep 22 23:22:21.802 INFO Completion from [0] id:390 status:true
Sep 22 23:22:21.802 INFO [391/752] Repair commands completed
Sep 22 23:22:21.802 INFO Pop front: ReconcileIO { id: ReconciliationId(391), op: ExtentReopen { repair_id: ReconciliationId(391), extent_id: 84 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.802 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.802 INFO [0] received reconcile message
Sep 22 23:22:21.802 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(391), op: ExtentReopen { repair_id: ReconciliationId(391), extent_id: 84 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.802 INFO [0] client ExtentReopen { repair_id: ReconciliationId(391), extent_id: 84 }
Sep 22 23:22:21.802 INFO [1] received reconcile message
Sep 22 23:22:21.802 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(391), op: ExtentReopen { repair_id: ReconciliationId(391), extent_id: 84 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.802 INFO [1] client ExtentReopen { repair_id: ReconciliationId(391), extent_id: 84 }
Sep 22 23:22:21.802 INFO [2] received reconcile message
Sep 22 23:22:21.802 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(391), op: ExtentReopen { repair_id: ReconciliationId(391), extent_id: 84 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.802 INFO [2] client ExtentReopen { repair_id: ReconciliationId(391), extent_id: 84 }
Sep 22 23:22:21.803 DEBG 391 Reopen extent 84
Sep 22 23:22:21.803 DEBG 391 Reopen extent 84
Sep 22 23:22:21.804 DEBG 391 Reopen extent 84
Sep 22 23:22:21.804 DEBG [2] It's time to notify for 391
Sep 22 23:22:21.804 INFO Completion from [2] id:391 status:true
Sep 22 23:22:21.804 INFO [392/752] Repair commands completed
Sep 22 23:22:21.804 INFO Pop front: ReconcileIO { id: ReconciliationId(392), op: ExtentFlush { repair_id: ReconciliationId(392), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.804 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.804 INFO [0] received reconcile message
Sep 22 23:22:21.804 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(392), op: ExtentFlush { repair_id: ReconciliationId(392), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.805 INFO [0] client ExtentFlush { repair_id: ReconciliationId(392), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.805 INFO [1] received reconcile message
Sep 22 23:22:21.805 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(392), op: ExtentFlush { repair_id: ReconciliationId(392), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.805 INFO [1] client ExtentFlush { repair_id: ReconciliationId(392), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.805 INFO [2] received reconcile message
Sep 22 23:22:21.805 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(392), op: ExtentFlush { repair_id: ReconciliationId(392), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.805 INFO [2] client ExtentFlush { repair_id: ReconciliationId(392), extent_id: 56, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.805 DEBG 392 Flush extent 56 with f:2 g:2
Sep 22 23:22:21.805 DEBG Flush just extent 56 with f:2 and g:2
Sep 22 23:22:21.805 DEBG [1] It's time to notify for 392
Sep 22 23:22:21.805 INFO Completion from [1] id:392 status:true
Sep 22 23:22:21.805 INFO [393/752] Repair commands completed
Sep 22 23:22:21.805 INFO Pop front: ReconcileIO { id: ReconciliationId(393), op: ExtentClose { repair_id: ReconciliationId(393), extent_id: 56 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.805 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.805 INFO [0] received reconcile message
Sep 22 23:22:21.805 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(393), op: ExtentClose { repair_id: ReconciliationId(393), extent_id: 56 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.805 INFO [0] client ExtentClose { repair_id: ReconciliationId(393), extent_id: 56 }
Sep 22 23:22:21.805 INFO [1] received reconcile message
Sep 22 23:22:21.805 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(393), op: ExtentClose { repair_id: ReconciliationId(393), extent_id: 56 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.805 INFO [1] client ExtentClose { repair_id: ReconciliationId(393), extent_id: 56 }
Sep 22 23:22:21.805 INFO [2] received reconcile message
Sep 22 23:22:21.805 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(393), op: ExtentClose { repair_id: ReconciliationId(393), extent_id: 56 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.805 INFO [2] client ExtentClose { repair_id: ReconciliationId(393), extent_id: 56 }
Sep 22 23:22:21.805 DEBG 393 Close extent 56
Sep 22 23:22:21.806 DEBG 393 Close extent 56
Sep 22 23:22:21.806 DEBG 393 Close extent 56
Sep 22 23:22:21.806 DEBG [2] It's time to notify for 393
Sep 22 23:22:21.806 INFO Completion from [2] id:393 status:true
Sep 22 23:22:21.806 INFO [394/752] Repair commands completed
Sep 22 23:22:21.806 INFO Pop front: ReconcileIO { id: ReconciliationId(394), op: ExtentRepair { repair_id: ReconciliationId(394), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.806 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.806 INFO [0] received reconcile message
Sep 22 23:22:21.806 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(394), op: ExtentRepair { repair_id: ReconciliationId(394), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.807 INFO [0] client ExtentRepair { repair_id: ReconciliationId(394), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.807 INFO [0] Sending repair request ReconciliationId(394)
Sep 22 23:22:21.807 INFO [1] received reconcile message
Sep 22 23:22:21.807 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(394), op: ExtentRepair { repair_id: ReconciliationId(394), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.807 INFO [1] client ExtentRepair { repair_id: ReconciliationId(394), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.807 INFO [1] No action required ReconciliationId(394)
Sep 22 23:22:21.807 INFO [2] received reconcile message
Sep 22 23:22:21.807 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(394), op: ExtentRepair { repair_id: ReconciliationId(394), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.807 INFO [2] client ExtentRepair { repair_id: ReconciliationId(394), extent_id: 56, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.807 INFO [2] No action required ReconciliationId(394)
Sep 22 23:22:21.807 DEBG 394 Repair extent 56 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.807 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/038.copy"
Sep 22 23:22:21.871 INFO accepted connection, remote_addr: 127.0.0.1:51239, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.871 TRCE incoming request, uri: /extent/56/files, method: GET, req_id: f2404592-5a05-45ec-a98e-467c9ad6a4da, remote_addr: 127.0.0.1:51239, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.872 INFO request completed, latency_us: 201, response_code: 200, uri: /extent/56/files, method: GET, req_id: f2404592-5a05-45ec-a98e-467c9ad6a4da, remote_addr: 127.0.0.1:51239, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.872 INFO eid:56 Found repair files: ["038", "038.db"]
Sep 22 23:22:21.872 TRCE incoming request, uri: /newextent/56/data, method: GET, req_id: 693c5484-7cb0-4dba-8f1e-a06d51646f49, remote_addr: 127.0.0.1:51239, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.872 INFO request completed, latency_us: 326, response_code: 200, uri: /newextent/56/data, method: GET, req_id: 693c5484-7cb0-4dba-8f1e-a06d51646f49, remote_addr: 127.0.0.1:51239, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.877 TRCE incoming request, uri: /newextent/56/db, method: GET, req_id: 0eb03cd7-6f4e-489b-a85c-3fd6c946c7ea, remote_addr: 127.0.0.1:51239, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.878 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/56/db, method: GET, req_id: 0eb03cd7-6f4e-489b-a85c-3fd6c946c7ea, remote_addr: 127.0.0.1:51239, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.879 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/038.copy" to "/tmp/downstairs-zrMnlo6G/00/000/038.replace"
Sep 22 23:22:21.879 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.880 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/038.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.880 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/038"
Sep 22 23:22:21.880 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/038.db"
Sep 22 23:22:21.880 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.880 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/038.replace" to "/tmp/downstairs-zrMnlo6G/00/000/038.completed"
Sep 22 23:22:21.880 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.880 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.880 DEBG [0] It's time to notify for 394
Sep 22 23:22:21.881 INFO Completion from [0] id:394 status:true
Sep 22 23:22:21.881 INFO [395/752] Repair commands completed
Sep 22 23:22:21.881 INFO Pop front: ReconcileIO { id: ReconciliationId(395), op: ExtentReopen { repair_id: ReconciliationId(395), extent_id: 56 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.881 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.881 INFO [0] received reconcile message
Sep 22 23:22:21.881 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(395), op: ExtentReopen { repair_id: ReconciliationId(395), extent_id: 56 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.881 INFO [0] client ExtentReopen { repair_id: ReconciliationId(395), extent_id: 56 }
Sep 22 23:22:21.881 INFO [1] received reconcile message
Sep 22 23:22:21.881 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(395), op: ExtentReopen { repair_id: ReconciliationId(395), extent_id: 56 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.881 INFO [1] client ExtentReopen { repair_id: ReconciliationId(395), extent_id: 56 }
Sep 22 23:22:21.881 INFO [2] received reconcile message
Sep 22 23:22:21.881 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(395), op: ExtentReopen { repair_id: ReconciliationId(395), extent_id: 56 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.881 INFO [2] client ExtentReopen { repair_id: ReconciliationId(395), extent_id: 56 }
Sep 22 23:22:21.881 DEBG 395 Reopen extent 56
Sep 22 23:22:21.882 DEBG 395 Reopen extent 56
Sep 22 23:22:21.882 DEBG 395 Reopen extent 56
Sep 22 23:22:21.883 DEBG [2] It's time to notify for 395
Sep 22 23:22:21.883 INFO Completion from [2] id:395 status:true
Sep 22 23:22:21.883 INFO [396/752] Repair commands completed
Sep 22 23:22:21.883 INFO Pop front: ReconcileIO { id: ReconciliationId(396), op: ExtentFlush { repair_id: ReconciliationId(396), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.883 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.883 INFO [0] received reconcile message
Sep 22 23:22:21.883 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(396), op: ExtentFlush { repair_id: ReconciliationId(396), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.883 INFO [0] client ExtentFlush { repair_id: ReconciliationId(396), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.883 INFO [1] received reconcile message
Sep 22 23:22:21.883 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(396), op: ExtentFlush { repair_id: ReconciliationId(396), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.883 INFO [1] client ExtentFlush { repair_id: ReconciliationId(396), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.883 INFO [2] received reconcile message
Sep 22 23:22:21.883 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(396), op: ExtentFlush { repair_id: ReconciliationId(396), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.883 INFO [2] client ExtentFlush { repair_id: ReconciliationId(396), extent_id: 94, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.883 DEBG 396 Flush extent 94 with f:2 g:2
Sep 22 23:22:21.883 DEBG Flush just extent 94 with f:2 and g:2
Sep 22 23:22:21.883 DEBG [1] It's time to notify for 396
Sep 22 23:22:21.883 INFO Completion from [1] id:396 status:true
Sep 22 23:22:21.883 INFO [397/752] Repair commands completed
Sep 22 23:22:21.883 INFO Pop front: ReconcileIO { id: ReconciliationId(397), op: ExtentClose { repair_id: ReconciliationId(397), extent_id: 94 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.883 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.883 INFO [0] received reconcile message
Sep 22 23:22:21.883 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(397), op: ExtentClose { repair_id: ReconciliationId(397), extent_id: 94 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.883 INFO [0] client ExtentClose { repair_id: ReconciliationId(397), extent_id: 94 }
Sep 22 23:22:21.883 INFO [1] received reconcile message
Sep 22 23:22:21.884 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(397), op: ExtentClose { repair_id: ReconciliationId(397), extent_id: 94 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.884 INFO [1] client ExtentClose { repair_id: ReconciliationId(397), extent_id: 94 }
Sep 22 23:22:21.884 INFO [2] received reconcile message
Sep 22 23:22:21.884 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(397), op: ExtentClose { repair_id: ReconciliationId(397), extent_id: 94 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.884 INFO [2] client ExtentClose { repair_id: ReconciliationId(397), extent_id: 94 }
Sep 22 23:22:21.884 DEBG 397 Close extent 94
Sep 22 23:22:21.884 DEBG 397 Close extent 94
Sep 22 23:22:21.884 DEBG 397 Close extent 94
Sep 22 23:22:21.885 DEBG [2] It's time to notify for 397
Sep 22 23:22:21.885 INFO Completion from [2] id:397 status:true
Sep 22 23:22:21.885 INFO [398/752] Repair commands completed
Sep 22 23:22:21.885 INFO Pop front: ReconcileIO { id: ReconciliationId(398), op: ExtentRepair { repair_id: ReconciliationId(398), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.885 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.885 INFO [0] received reconcile message
Sep 22 23:22:21.885 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(398), op: ExtentRepair { repair_id: ReconciliationId(398), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.885 INFO [0] client ExtentRepair { repair_id: ReconciliationId(398), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.885 INFO [0] Sending repair request ReconciliationId(398)
Sep 22 23:22:21.885 INFO [1] received reconcile message
Sep 22 23:22:21.885 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(398), op: ExtentRepair { repair_id: ReconciliationId(398), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.885 INFO [1] client ExtentRepair { repair_id: ReconciliationId(398), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.885 INFO [1] No action required ReconciliationId(398)
Sep 22 23:22:21.885 INFO [2] received reconcile message
Sep 22 23:22:21.885 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(398), op: ExtentRepair { repair_id: ReconciliationId(398), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.885 INFO [2] client ExtentRepair { repair_id: ReconciliationId(398), extent_id: 94, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.885 INFO [2] No action required ReconciliationId(398)
Sep 22 23:22:21.885 DEBG 398 Repair extent 94 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.885 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/05E.copy"
Sep 22 23:22:21.949 INFO accepted connection, remote_addr: 127.0.0.1:32831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.950 TRCE incoming request, uri: /extent/94/files, method: GET, req_id: 5ae072b3-ef43-40db-aeca-9f3eabcf43f8, remote_addr: 127.0.0.1:32831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.950 INFO request completed, latency_us: 236, response_code: 200, uri: /extent/94/files, method: GET, req_id: 5ae072b3-ef43-40db-aeca-9f3eabcf43f8, remote_addr: 127.0.0.1:32831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.950 INFO eid:94 Found repair files: ["05E", "05E.db"]
Sep 22 23:22:21.950 TRCE incoming request, uri: /newextent/94/data, method: GET, req_id: f30b7809-59e9-47ea-9ef3-03dac011f09c, remote_addr: 127.0.0.1:32831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.951 INFO request completed, latency_us: 326, response_code: 200, uri: /newextent/94/data, method: GET, req_id: f30b7809-59e9-47ea-9ef3-03dac011f09c, remote_addr: 127.0.0.1:32831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.956 TRCE incoming request, uri: /newextent/94/db, method: GET, req_id: 7cb4c93d-cb6e-43dc-9748-091cd235c884, remote_addr: 127.0.0.1:32831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.956 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/94/db, method: GET, req_id: 7cb4c93d-cb6e-43dc-9748-091cd235c884, remote_addr: 127.0.0.1:32831, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:21.957 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/05E.copy" to "/tmp/downstairs-zrMnlo6G/00/000/05E.replace"
Sep 22 23:22:21.957 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.958 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/05E.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.958 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05E"
Sep 22 23:22:21.958 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05E.db"
Sep 22 23:22:21.958 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.958 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/05E.replace" to "/tmp/downstairs-zrMnlo6G/00/000/05E.completed"
Sep 22 23:22:21.959 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.959 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:21.959 DEBG [0] It's time to notify for 398
Sep 22 23:22:21.959 INFO Completion from [0] id:398 status:true
Sep 22 23:22:21.959 INFO [399/752] Repair commands completed
Sep 22 23:22:21.959 INFO Pop front: ReconcileIO { id: ReconciliationId(399), op: ExtentReopen { repair_id: ReconciliationId(399), extent_id: 94 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.959 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.959 INFO [0] received reconcile message
Sep 22 23:22:21.959 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(399), op: ExtentReopen { repair_id: ReconciliationId(399), extent_id: 94 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.959 INFO [0] client ExtentReopen { repair_id: ReconciliationId(399), extent_id: 94 }
Sep 22 23:22:21.959 INFO [1] received reconcile message
Sep 22 23:22:21.959 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(399), op: ExtentReopen { repair_id: ReconciliationId(399), extent_id: 94 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.959 INFO [1] client ExtentReopen { repair_id: ReconciliationId(399), extent_id: 94 }
Sep 22 23:22:21.959 INFO [2] received reconcile message
Sep 22 23:22:21.959 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(399), op: ExtentReopen { repair_id: ReconciliationId(399), extent_id: 94 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.959 INFO [2] client ExtentReopen { repair_id: ReconciliationId(399), extent_id: 94 }
Sep 22 23:22:21.959 DEBG 399 Reopen extent 94
Sep 22 23:22:21.960 DEBG 399 Reopen extent 94
Sep 22 23:22:21.961 DEBG 399 Reopen extent 94
Sep 22 23:22:21.961 DEBG [2] It's time to notify for 399
Sep 22 23:22:21.961 INFO Completion from [2] id:399 status:true
Sep 22 23:22:21.961 INFO [400/752] Repair commands completed
Sep 22 23:22:21.961 INFO Pop front: ReconcileIO { id: ReconciliationId(400), op: ExtentFlush { repair_id: ReconciliationId(400), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.961 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.961 INFO [0] received reconcile message
Sep 22 23:22:21.961 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(400), op: ExtentFlush { repair_id: ReconciliationId(400), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.961 INFO [0] client ExtentFlush { repair_id: ReconciliationId(400), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.961 INFO [1] received reconcile message
Sep 22 23:22:21.961 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(400), op: ExtentFlush { repair_id: ReconciliationId(400), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:21.961 INFO [1] client ExtentFlush { repair_id: ReconciliationId(400), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.962 INFO [2] received reconcile message
Sep 22 23:22:21.962 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(400), op: ExtentFlush { repair_id: ReconciliationId(400), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.962 INFO [2] client ExtentFlush { repair_id: ReconciliationId(400), extent_id: 93, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:21.962 DEBG 400 Flush extent 93 with f:2 g:2
Sep 22 23:22:21.962 DEBG Flush just extent 93 with f:2 and g:2
Sep 22 23:22:21.962 DEBG [1] It's time to notify for 400
Sep 22 23:22:21.962 INFO Completion from [1] id:400 status:true
Sep 22 23:22:21.962 INFO [401/752] Repair commands completed
Sep 22 23:22:21.962 INFO Pop front: ReconcileIO { id: ReconciliationId(401), op: ExtentClose { repair_id: ReconciliationId(401), extent_id: 93 }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.962 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.962 INFO [0] received reconcile message
Sep 22 23:22:21.962 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(401), op: ExtentClose { repair_id: ReconciliationId(401), extent_id: 93 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.962 INFO [0] client ExtentClose { repair_id: ReconciliationId(401), extent_id: 93 }
Sep 22 23:22:21.962 INFO [1] received reconcile message
Sep 22 23:22:21.962 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(401), op: ExtentClose { repair_id: ReconciliationId(401), extent_id: 93 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.962 INFO [1] client ExtentClose { repair_id: ReconciliationId(401), extent_id: 93 }
Sep 22 23:22:21.962 INFO [2] received reconcile message
Sep 22 23:22:21.962 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(401), op: ExtentClose { repair_id: ReconciliationId(401), extent_id: 93 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:21.962 INFO [2] client ExtentClose { repair_id: ReconciliationId(401), extent_id: 93 }
Sep 22 23:22:21.962 DEBG 401 Close extent 93
Sep 22 23:22:21.963 DEBG 401 Close extent 93
Sep 22 23:22:21.963 DEBG 401 Close extent 93
Sep 22 23:22:21.963 DEBG [2] It's time to notify for 401
Sep 22 23:22:21.963 INFO Completion from [2] id:401 status:true
Sep 22 23:22:21.963 INFO [402/752] Repair commands completed
Sep 22 23:22:21.963 INFO Pop front: ReconcileIO { id: ReconciliationId(402), op: ExtentRepair { repair_id: ReconciliationId(402), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:21.963 INFO Sent repair work, now wait for resp
Sep 22 23:22:21.963 INFO [0] received reconcile message
Sep 22 23:22:21.963 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(402), op: ExtentRepair { repair_id: ReconciliationId(402), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:21.963 INFO [0] client ExtentRepair { repair_id: ReconciliationId(402), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.963 INFO [0] Sending repair request ReconciliationId(402)
Sep 22 23:22:21.964 INFO [1] received reconcile message
Sep 22 23:22:21.964 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(402), op: ExtentRepair { repair_id: ReconciliationId(402), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:21.964 INFO [1] client ExtentRepair { repair_id: ReconciliationId(402), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.964 INFO [1] No action required ReconciliationId(402)
Sep 22 23:22:21.964 INFO [2] received reconcile message
Sep 22 23:22:21.964 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(402), op: ExtentRepair { repair_id: ReconciliationId(402), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:21.964 INFO [2] client ExtentRepair { repair_id: ReconciliationId(402), extent_id: 93, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:21.964 INFO [2] No action required ReconciliationId(402)
Sep 22 23:22:21.964 DEBG 402 Repair extent 93 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:21.964 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/05D.copy"
Sep 22 23:22:22.029 INFO accepted connection, remote_addr: 127.0.0.1:39873, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.029 TRCE incoming request, uri: /extent/93/files, method: GET, req_id: 92e54ccf-7732-462e-8b41-d50464e746f5, remote_addr: 127.0.0.1:39873, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.029 INFO request completed, latency_us: 264, response_code: 200, uri: /extent/93/files, method: GET, req_id: 92e54ccf-7732-462e-8b41-d50464e746f5, remote_addr: 127.0.0.1:39873, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.030 INFO eid:93 Found repair files: ["05D", "05D.db"]
Sep 22 23:22:22.030 TRCE incoming request, uri: /newextent/93/data, method: GET, req_id: f4b98ba5-84e6-4edd-8da4-a7cf6828e059, remote_addr: 127.0.0.1:39873, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.030 INFO request completed, latency_us: 374, response_code: 200, uri: /newextent/93/data, method: GET, req_id: f4b98ba5-84e6-4edd-8da4-a7cf6828e059, remote_addr: 127.0.0.1:39873, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.035 TRCE incoming request, uri: /newextent/93/db, method: GET, req_id: 1d51a21c-3761-49ce-a8e6-e1fb9b24b5db, remote_addr: 127.0.0.1:39873, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.035 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/93/db, method: GET, req_id: 1d51a21c-3761-49ce-a8e6-e1fb9b24b5db, remote_addr: 127.0.0.1:39873, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.036 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/05D.copy" to "/tmp/downstairs-zrMnlo6G/00/000/05D.replace"
Sep 22 23:22:22.036 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.037 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/05D.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.038 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05D"
Sep 22 23:22:22.038 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05D.db"
Sep 22 23:22:22.038 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.038 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/05D.replace" to "/tmp/downstairs-zrMnlo6G/00/000/05D.completed"
Sep 22 23:22:22.038 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.038 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.038 DEBG [0] It's time to notify for 402
Sep 22 23:22:22.038 INFO Completion from [0] id:402 status:true
Sep 22 23:22:22.038 INFO [403/752] Repair commands completed
Sep 22 23:22:22.038 INFO Pop front: ReconcileIO { id: ReconciliationId(403), op: ExtentReopen { repair_id: ReconciliationId(403), extent_id: 93 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.038 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.038 INFO [0] received reconcile message
Sep 22 23:22:22.038 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(403), op: ExtentReopen { repair_id: ReconciliationId(403), extent_id: 93 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.038 INFO [0] client ExtentReopen { repair_id: ReconciliationId(403), extent_id: 93 }
Sep 22 23:22:22.039 INFO [1] received reconcile message
Sep 22 23:22:22.039 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(403), op: ExtentReopen { repair_id: ReconciliationId(403), extent_id: 93 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.039 INFO [1] client ExtentReopen { repair_id: ReconciliationId(403), extent_id: 93 }
Sep 22 23:22:22.039 INFO [2] received reconcile message
Sep 22 23:22:22.039 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(403), op: ExtentReopen { repair_id: ReconciliationId(403), extent_id: 93 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.039 INFO [2] client ExtentReopen { repair_id: ReconciliationId(403), extent_id: 93 }
Sep 22 23:22:22.039 DEBG 403 Reopen extent 93
Sep 22 23:22:22.040 DEBG 403 Reopen extent 93
Sep 22 23:22:22.040 DEBG 403 Reopen extent 93
Sep 22 23:22:22.041 DEBG [2] It's time to notify for 403
Sep 22 23:22:22.041 INFO Completion from [2] id:403 status:true
Sep 22 23:22:22.041 INFO [404/752] Repair commands completed
Sep 22 23:22:22.041 INFO Pop front: ReconcileIO { id: ReconciliationId(404), op: ExtentFlush { repair_id: ReconciliationId(404), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.041 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.041 INFO [0] received reconcile message
Sep 22 23:22:22.041 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(404), op: ExtentFlush { repair_id: ReconciliationId(404), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.041 INFO [0] client ExtentFlush { repair_id: ReconciliationId(404), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.041 INFO [1] received reconcile message
Sep 22 23:22:22.041 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(404), op: ExtentFlush { repair_id: ReconciliationId(404), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.041 INFO [1] client ExtentFlush { repair_id: ReconciliationId(404), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.041 INFO [2] received reconcile message
Sep 22 23:22:22.041 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(404), op: ExtentFlush { repair_id: ReconciliationId(404), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.041 INFO [2] client ExtentFlush { repair_id: ReconciliationId(404), extent_id: 44, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.041 DEBG 404 Flush extent 44 with f:2 g:2
Sep 22 23:22:22.041 DEBG Flush just extent 44 with f:2 and g:2
Sep 22 23:22:22.041 DEBG [1] It's time to notify for 404
Sep 22 23:22:22.041 INFO Completion from [1] id:404 status:true
Sep 22 23:22:22.041 INFO [405/752] Repair commands completed
Sep 22 23:22:22.041 INFO Pop front: ReconcileIO { id: ReconciliationId(405), op: ExtentClose { repair_id: ReconciliationId(405), extent_id: 44 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.041 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.041 INFO [0] received reconcile message
Sep 22 23:22:22.041 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(405), op: ExtentClose { repair_id: ReconciliationId(405), extent_id: 44 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.041 INFO [0] client ExtentClose { repair_id: ReconciliationId(405), extent_id: 44 }
Sep 22 23:22:22.042 INFO [1] received reconcile message
Sep 22 23:22:22.042 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(405), op: ExtentClose { repair_id: ReconciliationId(405), extent_id: 44 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.042 INFO [1] client ExtentClose { repair_id: ReconciliationId(405), extent_id: 44 }
Sep 22 23:22:22.042 INFO [2] received reconcile message
Sep 22 23:22:22.042 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(405), op: ExtentClose { repair_id: ReconciliationId(405), extent_id: 44 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.042 INFO [2] client ExtentClose { repair_id: ReconciliationId(405), extent_id: 44 }
Sep 22 23:22:22.042 DEBG 405 Close extent 44
Sep 22 23:22:22.042 DEBG 405 Close extent 44
Sep 22 23:22:22.042 DEBG 405 Close extent 44
Sep 22 23:22:22.043 DEBG [2] It's time to notify for 405
Sep 22 23:22:22.043 INFO Completion from [2] id:405 status:true
Sep 22 23:22:22.043 INFO [406/752] Repair commands completed
Sep 22 23:22:22.043 INFO Pop front: ReconcileIO { id: ReconciliationId(406), op: ExtentRepair { repair_id: ReconciliationId(406), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.043 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.043 INFO [0] received reconcile message
Sep 22 23:22:22.043 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(406), op: ExtentRepair { repair_id: ReconciliationId(406), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.043 INFO [0] client ExtentRepair { repair_id: ReconciliationId(406), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.043 INFO [0] Sending repair request ReconciliationId(406)
Sep 22 23:22:22.043 INFO [1] received reconcile message
Sep 22 23:22:22.043 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(406), op: ExtentRepair { repair_id: ReconciliationId(406), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.043 INFO [1] client ExtentRepair { repair_id: ReconciliationId(406), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.043 INFO [1] No action required ReconciliationId(406)
Sep 22 23:22:22.043 INFO [2] received reconcile message
Sep 22 23:22:22.043 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(406), op: ExtentRepair { repair_id: ReconciliationId(406), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.043 INFO [2] client ExtentRepair { repair_id: ReconciliationId(406), extent_id: 44, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.043 INFO [2] No action required ReconciliationId(406)
Sep 22 23:22:22.043 DEBG 406 Repair extent 44 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.043 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/02C.copy"
Sep 22 23:22:22.107 INFO accepted connection, remote_addr: 127.0.0.1:47489, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.107 TRCE incoming request, uri: /extent/44/files, method: GET, req_id: 54ed40bf-ac6b-448e-bf6a-07b93bc3eeb4, remote_addr: 127.0.0.1:47489, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.108 INFO request completed, latency_us: 282, response_code: 200, uri: /extent/44/files, method: GET, req_id: 54ed40bf-ac6b-448e-bf6a-07b93bc3eeb4, remote_addr: 127.0.0.1:47489, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.108 INFO eid:44 Found repair files: ["02C", "02C.db"]
Sep 22 23:22:22.108 TRCE incoming request, uri: /newextent/44/data, method: GET, req_id: d7373006-990b-40ff-850c-14658e02a461, remote_addr: 127.0.0.1:47489, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.109 INFO request completed, latency_us: 350, response_code: 200, uri: /newextent/44/data, method: GET, req_id: d7373006-990b-40ff-850c-14658e02a461, remote_addr: 127.0.0.1:47489, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.113 TRCE incoming request, uri: /newextent/44/db, method: GET, req_id: 35fcbe58-b60f-4a71-a921-a4b338070505, remote_addr: 127.0.0.1:47489, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.114 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/44/db, method: GET, req_id: 35fcbe58-b60f-4a71-a921-a4b338070505, remote_addr: 127.0.0.1:47489, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.115 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/02C.copy" to "/tmp/downstairs-zrMnlo6G/00/000/02C.replace"
Sep 22 23:22:22.115 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.116 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/02C.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.116 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02C"
Sep 22 23:22:22.116 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02C.db"
Sep 22 23:22:22.116 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.116 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/02C.replace" to "/tmp/downstairs-zrMnlo6G/00/000/02C.completed"
Sep 22 23:22:22.116 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.116 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.117 DEBG [0] It's time to notify for 406
Sep 22 23:22:22.117 INFO Completion from [0] id:406 status:true
Sep 22 23:22:22.117 INFO [407/752] Repair commands completed
Sep 22 23:22:22.117 INFO Pop front: ReconcileIO { id: ReconciliationId(407), op: ExtentReopen { repair_id: ReconciliationId(407), extent_id: 44 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.117 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.117 INFO [0] received reconcile message
Sep 22 23:22:22.117 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(407), op: ExtentReopen { repair_id: ReconciliationId(407), extent_id: 44 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.117 INFO [0] client ExtentReopen { repair_id: ReconciliationId(407), extent_id: 44 }
Sep 22 23:22:22.117 INFO [1] received reconcile message
Sep 22 23:22:22.117 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(407), op: ExtentReopen { repair_id: ReconciliationId(407), extent_id: 44 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.117 INFO [1] client ExtentReopen { repair_id: ReconciliationId(407), extent_id: 44 }
Sep 22 23:22:22.117 INFO [2] received reconcile message
Sep 22 23:22:22.117 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(407), op: ExtentReopen { repair_id: ReconciliationId(407), extent_id: 44 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.117 INFO [2] client ExtentReopen { repair_id: ReconciliationId(407), extent_id: 44 }
Sep 22 23:22:22.117 DEBG 407 Reopen extent 44
Sep 22 23:22:22.118 DEBG 407 Reopen extent 44
Sep 22 23:22:22.119 DEBG 407 Reopen extent 44
Sep 22 23:22:22.119 DEBG [2] It's time to notify for 407
Sep 22 23:22:22.119 INFO Completion from [2] id:407 status:true
Sep 22 23:22:22.119 INFO [408/752] Repair commands completed
Sep 22 23:22:22.119 INFO Pop front: ReconcileIO { id: ReconciliationId(408), op: ExtentFlush { repair_id: ReconciliationId(408), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.119 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.119 INFO [0] received reconcile message
Sep 22 23:22:22.120 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(408), op: ExtentFlush { repair_id: ReconciliationId(408), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.120 INFO [0] client ExtentFlush { repair_id: ReconciliationId(408), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.120 INFO [1] received reconcile message
Sep 22 23:22:22.120 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(408), op: ExtentFlush { repair_id: ReconciliationId(408), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.120 INFO [1] client ExtentFlush { repair_id: ReconciliationId(408), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.120 INFO [2] received reconcile message
Sep 22 23:22:22.120 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(408), op: ExtentFlush { repair_id: ReconciliationId(408), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.120 INFO [2] client ExtentFlush { repair_id: ReconciliationId(408), extent_id: 45, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.120 DEBG 408 Flush extent 45 with f:2 g:2
Sep 22 23:22:22.120 DEBG Flush just extent 45 with f:2 and g:2
Sep 22 23:22:22.120 DEBG [1] It's time to notify for 408
Sep 22 23:22:22.120 INFO Completion from [1] id:408 status:true
Sep 22 23:22:22.120 INFO [409/752] Repair commands completed
Sep 22 23:22:22.120 INFO Pop front: ReconcileIO { id: ReconciliationId(409), op: ExtentClose { repair_id: ReconciliationId(409), extent_id: 45 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.120 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.120 INFO [0] received reconcile message
Sep 22 23:22:22.120 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(409), op: ExtentClose { repair_id: ReconciliationId(409), extent_id: 45 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.120 INFO [0] client ExtentClose { repair_id: ReconciliationId(409), extent_id: 45 }
Sep 22 23:22:22.120 INFO [1] received reconcile message
Sep 22 23:22:22.120 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(409), op: ExtentClose { repair_id: ReconciliationId(409), extent_id: 45 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.120 INFO [1] client ExtentClose { repair_id: ReconciliationId(409), extent_id: 45 }
Sep 22 23:22:22.120 INFO [2] received reconcile message
Sep 22 23:22:22.120 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(409), op: ExtentClose { repair_id: ReconciliationId(409), extent_id: 45 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.120 INFO [2] client ExtentClose { repair_id: ReconciliationId(409), extent_id: 45 }
Sep 22 23:22:22.120 DEBG 409 Close extent 45
Sep 22 23:22:22.121 DEBG 409 Close extent 45
Sep 22 23:22:22.121 DEBG 409 Close extent 45
Sep 22 23:22:22.121 DEBG [2] It's time to notify for 409
Sep 22 23:22:22.121 INFO Completion from [2] id:409 status:true
Sep 22 23:22:22.121 INFO [410/752] Repair commands completed
Sep 22 23:22:22.121 INFO Pop front: ReconcileIO { id: ReconciliationId(410), op: ExtentRepair { repair_id: ReconciliationId(410), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.121 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.122 INFO [0] received reconcile message
Sep 22 23:22:22.122 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(410), op: ExtentRepair { repair_id: ReconciliationId(410), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.122 INFO [0] client ExtentRepair { repair_id: ReconciliationId(410), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.122 INFO [0] Sending repair request ReconciliationId(410)
Sep 22 23:22:22.122 INFO [1] received reconcile message
Sep 22 23:22:22.122 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(410), op: ExtentRepair { repair_id: ReconciliationId(410), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.122 INFO [1] client ExtentRepair { repair_id: ReconciliationId(410), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.122 INFO [1] No action required ReconciliationId(410)
Sep 22 23:22:22.122 INFO [2] received reconcile message
Sep 22 23:22:22.122 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(410), op: ExtentRepair { repair_id: ReconciliationId(410), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.122 INFO [2] client ExtentRepair { repair_id: ReconciliationId(410), extent_id: 45, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.122 INFO [2] No action required ReconciliationId(410)
Sep 22 23:22:22.122 DEBG 410 Repair extent 45 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.122 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/02D.copy"
Sep 22 23:22:22.146 DEBG up_ds_listen was notified
Sep 22 23:22:22.146 DEBG up_ds_listen process 1084
Sep 22 23:22:22.146 DEBG [A] ack job 1084:85, : downstairs
Sep 22 23:22:22.146 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:22.146 DEBG Flush     :1082 extent_limit None deps:[JobId(1081), JobId(1080)] res:true f:30 g:1
Sep 22 23:22:22.152 DEBG Read      :1083 deps:[JobId(1082)] res:true
Sep 22 23:22:22.173 DEBG [rc] retire 1082 clears [JobId(1081), JobId(1082)], : downstairs
Sep 22 23:22:22.176 INFO [lossy] sleeping 1 second
Sep 22 23:22:22.187 INFO accepted connection, remote_addr: 127.0.0.1:64850, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.188 TRCE incoming request, uri: /extent/45/files, method: GET, req_id: f5aed8c7-4314-4fda-8d73-97115e9581ca, remote_addr: 127.0.0.1:64850, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.188 INFO request completed, latency_us: 264, response_code: 200, uri: /extent/45/files, method: GET, req_id: f5aed8c7-4314-4fda-8d73-97115e9581ca, remote_addr: 127.0.0.1:64850, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.188 INFO eid:45 Found repair files: ["02D", "02D.db"]
Sep 22 23:22:22.189 TRCE incoming request, uri: /newextent/45/data, method: GET, req_id: 6b98c4e4-1d47-4e98-b2f5-a8b6803c4e4c, remote_addr: 127.0.0.1:64850, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.189 INFO request completed, latency_us: 345, response_code: 200, uri: /newextent/45/data, method: GET, req_id: 6b98c4e4-1d47-4e98-b2f5-a8b6803c4e4c, remote_addr: 127.0.0.1:64850, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.194 TRCE incoming request, uri: /newextent/45/db, method: GET, req_id: b059bdd3-383b-4827-ae6d-d1fbb3b2be02, remote_addr: 127.0.0.1:64850, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.194 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/45/db, method: GET, req_id: b059bdd3-383b-4827-ae6d-d1fbb3b2be02, remote_addr: 127.0.0.1:64850, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.195 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/02D.copy" to "/tmp/downstairs-zrMnlo6G/00/000/02D.replace"
Sep 22 23:22:22.195 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.196 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/02D.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.196 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02D"
Sep 22 23:22:22.196 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02D.db"
Sep 22 23:22:22.196 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.196 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/02D.replace" to "/tmp/downstairs-zrMnlo6G/00/000/02D.completed"
Sep 22 23:22:22.196 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.197 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.197 DEBG [0] It's time to notify for 410
Sep 22 23:22:22.197 INFO Completion from [0] id:410 status:true
Sep 22 23:22:22.197 INFO [411/752] Repair commands completed
Sep 22 23:22:22.197 INFO Pop front: ReconcileIO { id: ReconciliationId(411), op: ExtentReopen { repair_id: ReconciliationId(411), extent_id: 45 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.197 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.197 INFO [0] received reconcile message
Sep 22 23:22:22.197 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(411), op: ExtentReopen { repair_id: ReconciliationId(411), extent_id: 45 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.197 INFO [0] client ExtentReopen { repair_id: ReconciliationId(411), extent_id: 45 }
Sep 22 23:22:22.197 INFO [1] received reconcile message
Sep 22 23:22:22.197 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(411), op: ExtentReopen { repair_id: ReconciliationId(411), extent_id: 45 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.197 INFO [1] client ExtentReopen { repair_id: ReconciliationId(411), extent_id: 45 }
Sep 22 23:22:22.197 INFO [2] received reconcile message
Sep 22 23:22:22.197 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(411), op: ExtentReopen { repair_id: ReconciliationId(411), extent_id: 45 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.197 INFO [2] client ExtentReopen { repair_id: ReconciliationId(411), extent_id: 45 }
Sep 22 23:22:22.197 DEBG 411 Reopen extent 45
Sep 22 23:22:22.198 DEBG 411 Reopen extent 45
Sep 22 23:22:22.199 DEBG 411 Reopen extent 45
Sep 22 23:22:22.199 DEBG [2] It's time to notify for 411
Sep 22 23:22:22.199 INFO Completion from [2] id:411 status:true
Sep 22 23:22:22.199 INFO [412/752] Repair commands completed
Sep 22 23:22:22.199 INFO Pop front: ReconcileIO { id: ReconciliationId(412), op: ExtentFlush { repair_id: ReconciliationId(412), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.199 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.199 INFO [0] received reconcile message
Sep 22 23:22:22.199 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(412), op: ExtentFlush { repair_id: ReconciliationId(412), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.200 INFO [0] client ExtentFlush { repair_id: ReconciliationId(412), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.200 INFO [1] received reconcile message
Sep 22 23:22:22.200 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(412), op: ExtentFlush { repair_id: ReconciliationId(412), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.200 INFO [1] client ExtentFlush { repair_id: ReconciliationId(412), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.200 INFO [2] received reconcile message
Sep 22 23:22:22.200 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(412), op: ExtentFlush { repair_id: ReconciliationId(412), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.200 INFO [2] client ExtentFlush { repair_id: ReconciliationId(412), extent_id: 115, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.200 DEBG 412 Flush extent 115 with f:2 g:2
Sep 22 23:22:22.200 DEBG Flush just extent 115 with f:2 and g:2
Sep 22 23:22:22.200 DEBG [1] It's time to notify for 412
Sep 22 23:22:22.200 INFO Completion from [1] id:412 status:true
Sep 22 23:22:22.200 INFO [413/752] Repair commands completed
Sep 22 23:22:22.200 INFO Pop front: ReconcileIO { id: ReconciliationId(413), op: ExtentClose { repair_id: ReconciliationId(413), extent_id: 115 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.200 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.200 INFO [0] received reconcile message
Sep 22 23:22:22.200 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(413), op: ExtentClose { repair_id: ReconciliationId(413), extent_id: 115 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.200 INFO [0] client ExtentClose { repair_id: ReconciliationId(413), extent_id: 115 }
Sep 22 23:22:22.200 INFO [1] received reconcile message
Sep 22 23:22:22.200 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(413), op: ExtentClose { repair_id: ReconciliationId(413), extent_id: 115 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.200 INFO [1] client ExtentClose { repair_id: ReconciliationId(413), extent_id: 115 }
Sep 22 23:22:22.200 INFO [2] received reconcile message
Sep 22 23:22:22.200 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(413), op: ExtentClose { repair_id: ReconciliationId(413), extent_id: 115 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.200 INFO [2] client ExtentClose { repair_id: ReconciliationId(413), extent_id: 115 }
Sep 22 23:22:22.200 DEBG 413 Close extent 115
Sep 22 23:22:22.201 DEBG 413 Close extent 115
Sep 22 23:22:22.201 DEBG 413 Close extent 115
Sep 22 23:22:22.201 DEBG [2] It's time to notify for 413
Sep 22 23:22:22.201 INFO Completion from [2] id:413 status:true
Sep 22 23:22:22.201 INFO [414/752] Repair commands completed
Sep 22 23:22:22.201 INFO Pop front: ReconcileIO { id: ReconciliationId(414), op: ExtentRepair { repair_id: ReconciliationId(414), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.201 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.202 INFO [0] received reconcile message
Sep 22 23:22:22.202 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(414), op: ExtentRepair { repair_id: ReconciliationId(414), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.202 INFO [0] client ExtentRepair { repair_id: ReconciliationId(414), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.202 INFO [0] Sending repair request ReconciliationId(414)
Sep 22 23:22:22.202 INFO [1] received reconcile message
Sep 22 23:22:22.202 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(414), op: ExtentRepair { repair_id: ReconciliationId(414), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.202 INFO [1] client ExtentRepair { repair_id: ReconciliationId(414), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.202 INFO [1] No action required ReconciliationId(414)
Sep 22 23:22:22.202 INFO [2] received reconcile message
Sep 22 23:22:22.202 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(414), op: ExtentRepair { repair_id: ReconciliationId(414), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.202 INFO [2] client ExtentRepair { repair_id: ReconciliationId(414), extent_id: 115, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.202 INFO [2] No action required ReconciliationId(414)
Sep 22 23:22:22.202 DEBG 414 Repair extent 115 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.202 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/073.copy"
Sep 22 23:22:22.265 INFO accepted connection, remote_addr: 127.0.0.1:58736, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.265 TRCE incoming request, uri: /extent/115/files, method: GET, req_id: 4aa6657a-e91a-49b7-bc25-aebfd9e854d4, remote_addr: 127.0.0.1:58736, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.266 INFO request completed, latency_us: 209, response_code: 200, uri: /extent/115/files, method: GET, req_id: 4aa6657a-e91a-49b7-bc25-aebfd9e854d4, remote_addr: 127.0.0.1:58736, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.266 INFO eid:115 Found repair files: ["073", "073.db"]
Sep 22 23:22:22.266 TRCE incoming request, uri: /newextent/115/data, method: GET, req_id: 2cacbc77-5007-40f1-9550-92f52cfa09af, remote_addr: 127.0.0.1:58736, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.267 INFO request completed, latency_us: 311, response_code: 200, uri: /newextent/115/data, method: GET, req_id: 2cacbc77-5007-40f1-9550-92f52cfa09af, remote_addr: 127.0.0.1:58736, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.271 TRCE incoming request, uri: /newextent/115/db, method: GET, req_id: fe5b4061-9199-4095-b921-361ed976f001, remote_addr: 127.0.0.1:58736, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.272 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/115/db, method: GET, req_id: fe5b4061-9199-4095-b921-361ed976f001, remote_addr: 127.0.0.1:58736, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.273 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/073.copy" to "/tmp/downstairs-zrMnlo6G/00/000/073.replace"
Sep 22 23:22:22.273 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.274 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/073.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.274 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/073"
Sep 22 23:22:22.274 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/073.db"
Sep 22 23:22:22.274 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.274 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/073.replace" to "/tmp/downstairs-zrMnlo6G/00/000/073.completed"
Sep 22 23:22:22.274 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.274 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.274 DEBG [0] It's time to notify for 414
Sep 22 23:22:22.274 INFO Completion from [0] id:414 status:true
Sep 22 23:22:22.274 INFO [415/752] Repair commands completed
Sep 22 23:22:22.274 INFO Pop front: ReconcileIO { id: ReconciliationId(415), op: ExtentReopen { repair_id: ReconciliationId(415), extent_id: 115 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.274 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.274 INFO [0] received reconcile message
Sep 22 23:22:22.274 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(415), op: ExtentReopen { repair_id: ReconciliationId(415), extent_id: 115 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.275 INFO [0] client ExtentReopen { repair_id: ReconciliationId(415), extent_id: 115 }
Sep 22 23:22:22.275 INFO [1] received reconcile message
Sep 22 23:22:22.275 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(415), op: ExtentReopen { repair_id: ReconciliationId(415), extent_id: 115 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.275 INFO [1] client ExtentReopen { repair_id: ReconciliationId(415), extent_id: 115 }
Sep 22 23:22:22.275 INFO [2] received reconcile message
Sep 22 23:22:22.275 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(415), op: ExtentReopen { repair_id: ReconciliationId(415), extent_id: 115 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.275 INFO [2] client ExtentReopen { repair_id: ReconciliationId(415), extent_id: 115 }
Sep 22 23:22:22.275 DEBG 415 Reopen extent 115
Sep 22 23:22:22.275 DEBG 415 Reopen extent 115
Sep 22 23:22:22.276 DEBG 415 Reopen extent 115
Sep 22 23:22:22.277 DEBG [2] It's time to notify for 415
Sep 22 23:22:22.277 INFO Completion from [2] id:415 status:true
Sep 22 23:22:22.277 INFO [416/752] Repair commands completed
Sep 22 23:22:22.277 INFO Pop front: ReconcileIO { id: ReconciliationId(416), op: ExtentFlush { repair_id: ReconciliationId(416), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.277 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.277 INFO [0] received reconcile message
Sep 22 23:22:22.277 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(416), op: ExtentFlush { repair_id: ReconciliationId(416), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.277 INFO [0] client ExtentFlush { repair_id: ReconciliationId(416), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.277 INFO [1] received reconcile message
Sep 22 23:22:22.277 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(416), op: ExtentFlush { repair_id: ReconciliationId(416), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.277 INFO [1] client ExtentFlush { repair_id: ReconciliationId(416), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.277 INFO [2] received reconcile message
Sep 22 23:22:22.277 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(416), op: ExtentFlush { repair_id: ReconciliationId(416), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.277 INFO [2] client ExtentFlush { repair_id: ReconciliationId(416), extent_id: 32, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.277 DEBG 416 Flush extent 32 with f:2 g:2
Sep 22 23:22:22.277 DEBG Flush just extent 32 with f:2 and g:2
Sep 22 23:22:22.277 DEBG [1] It's time to notify for 416
Sep 22 23:22:22.277 INFO Completion from [1] id:416 status:true
Sep 22 23:22:22.277 INFO [417/752] Repair commands completed
Sep 22 23:22:22.277 INFO Pop front: ReconcileIO { id: ReconciliationId(417), op: ExtentClose { repair_id: ReconciliationId(417), extent_id: 32 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.277 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.277 INFO [0] received reconcile message
Sep 22 23:22:22.277 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(417), op: ExtentClose { repair_id: ReconciliationId(417), extent_id: 32 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.277 INFO [0] client ExtentClose { repair_id: ReconciliationId(417), extent_id: 32 }
Sep 22 23:22:22.277 INFO [1] received reconcile message
Sep 22 23:22:22.277 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(417), op: ExtentClose { repair_id: ReconciliationId(417), extent_id: 32 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.277 INFO [1] client ExtentClose { repair_id: ReconciliationId(417), extent_id: 32 }
Sep 22 23:22:22.277 INFO [2] received reconcile message
Sep 22 23:22:22.277 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(417), op: ExtentClose { repair_id: ReconciliationId(417), extent_id: 32 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.277 INFO [2] client ExtentClose { repair_id: ReconciliationId(417), extent_id: 32 }
Sep 22 23:22:22.278 DEBG 417 Close extent 32
Sep 22 23:22:22.278 DEBG 417 Close extent 32
Sep 22 23:22:22.278 DEBG 417 Close extent 32
Sep 22 23:22:22.279 DEBG [2] It's time to notify for 417
Sep 22 23:22:22.279 INFO Completion from [2] id:417 status:true
Sep 22 23:22:22.279 INFO [418/752] Repair commands completed
Sep 22 23:22:22.279 INFO Pop front: ReconcileIO { id: ReconciliationId(418), op: ExtentRepair { repair_id: ReconciliationId(418), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.279 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.279 INFO [0] received reconcile message
Sep 22 23:22:22.279 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(418), op: ExtentRepair { repair_id: ReconciliationId(418), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.279 INFO [0] client ExtentRepair { repair_id: ReconciliationId(418), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.279 INFO [0] Sending repair request ReconciliationId(418)
Sep 22 23:22:22.279 INFO [1] received reconcile message
Sep 22 23:22:22.279 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(418), op: ExtentRepair { repair_id: ReconciliationId(418), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.279 INFO [1] client ExtentRepair { repair_id: ReconciliationId(418), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.279 INFO [1] No action required ReconciliationId(418)
Sep 22 23:22:22.279 INFO [2] received reconcile message
Sep 22 23:22:22.279 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(418), op: ExtentRepair { repair_id: ReconciliationId(418), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.279 INFO [2] client ExtentRepair { repair_id: ReconciliationId(418), extent_id: 32, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.279 INFO [2] No action required ReconciliationId(418)
Sep 22 23:22:22.279 DEBG 418 Repair extent 32 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.279 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/020.copy"
Sep 22 23:22:22.344 INFO accepted connection, remote_addr: 127.0.0.1:65445, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.344 TRCE incoming request, uri: /extent/32/files, method: GET, req_id: 3e786468-3ace-4608-9bb9-be0e1efd4ca4, remote_addr: 127.0.0.1:65445, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.344 INFO request completed, latency_us: 209, response_code: 200, uri: /extent/32/files, method: GET, req_id: 3e786468-3ace-4608-9bb9-be0e1efd4ca4, remote_addr: 127.0.0.1:65445, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.344 INFO eid:32 Found repair files: ["020", "020.db"]
Sep 22 23:22:22.345 TRCE incoming request, uri: /newextent/32/data, method: GET, req_id: b1e603a4-e6d0-4154-8a4d-ed999ce6747f, remote_addr: 127.0.0.1:65445, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.345 INFO request completed, latency_us: 265, response_code: 200, uri: /newextent/32/data, method: GET, req_id: b1e603a4-e6d0-4154-8a4d-ed999ce6747f, remote_addr: 127.0.0.1:65445, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.350 TRCE incoming request, uri: /newextent/32/db, method: GET, req_id: e03cf918-1563-48f4-93b5-34d018f48076, remote_addr: 127.0.0.1:65445, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.350 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/32/db, method: GET, req_id: e03cf918-1563-48f4-93b5-34d018f48076, remote_addr: 127.0.0.1:65445, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.351 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/020.copy" to "/tmp/downstairs-zrMnlo6G/00/000/020.replace"
Sep 22 23:22:22.351 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.352 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/020.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.352 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/020"
Sep 22 23:22:22.352 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/020.db"
Sep 22 23:22:22.352 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.352 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/020.replace" to "/tmp/downstairs-zrMnlo6G/00/000/020.completed"
Sep 22 23:22:22.352 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.352 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.353 DEBG [0] It's time to notify for 418
Sep 22 23:22:22.353 INFO Completion from [0] id:418 status:true
Sep 22 23:22:22.353 INFO [419/752] Repair commands completed
Sep 22 23:22:22.353 INFO Pop front: ReconcileIO { id: ReconciliationId(419), op: ExtentReopen { repair_id: ReconciliationId(419), extent_id: 32 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.353 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.353 INFO [0] received reconcile message
Sep 22 23:22:22.353 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(419), op: ExtentReopen { repair_id: ReconciliationId(419), extent_id: 32 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.353 INFO [0] client ExtentReopen { repair_id: ReconciliationId(419), extent_id: 32 }
Sep 22 23:22:22.353 INFO [1] received reconcile message
Sep 22 23:22:22.353 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(419), op: ExtentReopen { repair_id: ReconciliationId(419), extent_id: 32 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.353 INFO [1] client ExtentReopen { repair_id: ReconciliationId(419), extent_id: 32 }
Sep 22 23:22:22.353 INFO [2] received reconcile message
Sep 22 23:22:22.353 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(419), op: ExtentReopen { repair_id: ReconciliationId(419), extent_id: 32 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.353 INFO [2] client ExtentReopen { repair_id: ReconciliationId(419), extent_id: 32 }
Sep 22 23:22:22.353 DEBG 419 Reopen extent 32
Sep 22 23:22:22.354 DEBG 419 Reopen extent 32
Sep 22 23:22:22.354 DEBG 419 Reopen extent 32
Sep 22 23:22:22.355 DEBG [2] It's time to notify for 419
Sep 22 23:22:22.355 INFO Completion from [2] id:419 status:true
Sep 22 23:22:22.355 INFO [420/752] Repair commands completed
Sep 22 23:22:22.355 INFO Pop front: ReconcileIO { id: ReconciliationId(420), op: ExtentFlush { repair_id: ReconciliationId(420), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.355 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.355 INFO [0] received reconcile message
Sep 22 23:22:22.355 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(420), op: ExtentFlush { repair_id: ReconciliationId(420), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.355 INFO [0] client ExtentFlush { repair_id: ReconciliationId(420), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.355 INFO [1] received reconcile message
Sep 22 23:22:22.355 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(420), op: ExtentFlush { repair_id: ReconciliationId(420), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.355 INFO [1] client ExtentFlush { repair_id: ReconciliationId(420), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.355 INFO [2] received reconcile message
Sep 22 23:22:22.355 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(420), op: ExtentFlush { repair_id: ReconciliationId(420), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.355 INFO [2] client ExtentFlush { repair_id: ReconciliationId(420), extent_id: 141, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.355 DEBG 420 Flush extent 141 with f:2 g:2
Sep 22 23:22:22.355 DEBG Flush just extent 141 with f:2 and g:2
Sep 22 23:22:22.356 DEBG [1] It's time to notify for 420
Sep 22 23:22:22.356 INFO Completion from [1] id:420 status:true
Sep 22 23:22:22.356 INFO [421/752] Repair commands completed
Sep 22 23:22:22.356 INFO Pop front: ReconcileIO { id: ReconciliationId(421), op: ExtentClose { repair_id: ReconciliationId(421), extent_id: 141 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.356 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.356 INFO [0] received reconcile message
Sep 22 23:22:22.356 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(421), op: ExtentClose { repair_id: ReconciliationId(421), extent_id: 141 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.356 INFO [0] client ExtentClose { repair_id: ReconciliationId(421), extent_id: 141 }
Sep 22 23:22:22.356 INFO [1] received reconcile message
Sep 22 23:22:22.356 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(421), op: ExtentClose { repair_id: ReconciliationId(421), extent_id: 141 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.356 INFO [1] client ExtentClose { repair_id: ReconciliationId(421), extent_id: 141 }
Sep 22 23:22:22.356 INFO [2] received reconcile message
Sep 22 23:22:22.356 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(421), op: ExtentClose { repair_id: ReconciliationId(421), extent_id: 141 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.356 INFO [2] client ExtentClose { repair_id: ReconciliationId(421), extent_id: 141 }
Sep 22 23:22:22.356 DEBG 421 Close extent 141
Sep 22 23:22:22.356 DEBG 421 Close extent 141
Sep 22 23:22:22.357 DEBG 421 Close extent 141
Sep 22 23:22:22.357 DEBG [2] It's time to notify for 421
Sep 22 23:22:22.357 INFO Completion from [2] id:421 status:true
Sep 22 23:22:22.357 INFO [422/752] Repair commands completed
Sep 22 23:22:22.357 INFO Pop front: ReconcileIO { id: ReconciliationId(422), op: ExtentRepair { repair_id: ReconciliationId(422), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.357 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.357 INFO [0] received reconcile message
Sep 22 23:22:22.357 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(422), op: ExtentRepair { repair_id: ReconciliationId(422), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.357 INFO [0] client ExtentRepair { repair_id: ReconciliationId(422), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.357 INFO [0] Sending repair request ReconciliationId(422)
Sep 22 23:22:22.357 INFO [1] received reconcile message
Sep 22 23:22:22.357 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(422), op: ExtentRepair { repair_id: ReconciliationId(422), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.357 INFO [1] client ExtentRepair { repair_id: ReconciliationId(422), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.357 INFO [1] No action required ReconciliationId(422)
Sep 22 23:22:22.357 INFO [2] received reconcile message
Sep 22 23:22:22.357 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(422), op: ExtentRepair { repair_id: ReconciliationId(422), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.357 INFO [2] client ExtentRepair { repair_id: ReconciliationId(422), extent_id: 141, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.357 INFO [2] No action required ReconciliationId(422)
Sep 22 23:22:22.357 DEBG 422 Repair extent 141 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.358 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/08D.copy"
Sep 22 23:22:22.422 INFO accepted connection, remote_addr: 127.0.0.1:64214, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.423 TRCE incoming request, uri: /extent/141/files, method: GET, req_id: 66d607e5-b0b1-4e4d-ba8d-8eb1c7288ada, remote_addr: 127.0.0.1:64214, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.423 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/141/files, method: GET, req_id: 66d607e5-b0b1-4e4d-ba8d-8eb1c7288ada, remote_addr: 127.0.0.1:64214, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.423 INFO eid:141 Found repair files: ["08D", "08D.db"]
Sep 22 23:22:22.423 TRCE incoming request, uri: /newextent/141/data, method: GET, req_id: 8abbe60c-6f9f-42f4-9b40-0edc524f86d5, remote_addr: 127.0.0.1:64214, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.424 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/141/data, method: GET, req_id: 8abbe60c-6f9f-42f4-9b40-0edc524f86d5, remote_addr: 127.0.0.1:64214, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.428 TRCE incoming request, uri: /newextent/141/db, method: GET, req_id: c81c42cd-0acd-4f62-ac98-ab1adbc4885e, remote_addr: 127.0.0.1:64214, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.428 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/141/db, method: GET, req_id: c81c42cd-0acd-4f62-ac98-ab1adbc4885e, remote_addr: 127.0.0.1:64214, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.430 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/08D.copy" to "/tmp/downstairs-zrMnlo6G/00/000/08D.replace"
Sep 22 23:22:22.430 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.430 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/08D.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.431 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08D"
Sep 22 23:22:22.431 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08D.db"
Sep 22 23:22:22.431 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.431 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/08D.replace" to "/tmp/downstairs-zrMnlo6G/00/000/08D.completed"
Sep 22 23:22:22.431 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.431 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.431 DEBG [0] It's time to notify for 422
Sep 22 23:22:22.431 INFO Completion from [0] id:422 status:true
Sep 22 23:22:22.431 INFO [423/752] Repair commands completed
Sep 22 23:22:22.431 INFO Pop front: ReconcileIO { id: ReconciliationId(423), op: ExtentReopen { repair_id: ReconciliationId(423), extent_id: 141 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.431 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.431 INFO [0] received reconcile message
Sep 22 23:22:22.431 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(423), op: ExtentReopen { repair_id: ReconciliationId(423), extent_id: 141 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.431 INFO [0] client ExtentReopen { repair_id: ReconciliationId(423), extent_id: 141 }
Sep 22 23:22:22.431 INFO [1] received reconcile message
Sep 22 23:22:22.431 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(423), op: ExtentReopen { repair_id: ReconciliationId(423), extent_id: 141 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.431 INFO [1] client ExtentReopen { repair_id: ReconciliationId(423), extent_id: 141 }
Sep 22 23:22:22.432 INFO [2] received reconcile message
Sep 22 23:22:22.432 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(423), op: ExtentReopen { repair_id: ReconciliationId(423), extent_id: 141 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.432 INFO [2] client ExtentReopen { repair_id: ReconciliationId(423), extent_id: 141 }
Sep 22 23:22:22.432 DEBG 423 Reopen extent 141
Sep 22 23:22:22.432 DEBG 423 Reopen extent 141
Sep 22 23:22:22.433 DEBG 423 Reopen extent 141
Sep 22 23:22:22.433 DEBG [2] It's time to notify for 423
Sep 22 23:22:22.433 INFO Completion from [2] id:423 status:true
Sep 22 23:22:22.433 INFO [424/752] Repair commands completed
Sep 22 23:22:22.433 INFO Pop front: ReconcileIO { id: ReconciliationId(424), op: ExtentFlush { repair_id: ReconciliationId(424), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.433 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.434 INFO [0] received reconcile message
Sep 22 23:22:22.434 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(424), op: ExtentFlush { repair_id: ReconciliationId(424), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.434 INFO [0] client ExtentFlush { repair_id: ReconciliationId(424), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.434 INFO [1] received reconcile message
Sep 22 23:22:22.434 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(424), op: ExtentFlush { repair_id: ReconciliationId(424), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.434 INFO [1] client ExtentFlush { repair_id: ReconciliationId(424), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.434 INFO [2] received reconcile message
Sep 22 23:22:22.434 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(424), op: ExtentFlush { repair_id: ReconciliationId(424), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.434 INFO [2] client ExtentFlush { repair_id: ReconciliationId(424), extent_id: 9, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.434 DEBG 424 Flush extent 9 with f:2 g:2
Sep 22 23:22:22.434 DEBG Flush just extent 9 with f:2 and g:2
Sep 22 23:22:22.434 DEBG [1] It's time to notify for 424
Sep 22 23:22:22.434 INFO Completion from [1] id:424 status:true
Sep 22 23:22:22.434 INFO [425/752] Repair commands completed
Sep 22 23:22:22.434 INFO Pop front: ReconcileIO { id: ReconciliationId(425), op: ExtentClose { repair_id: ReconciliationId(425), extent_id: 9 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.434 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.434 INFO [0] received reconcile message
Sep 22 23:22:22.434 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(425), op: ExtentClose { repair_id: ReconciliationId(425), extent_id: 9 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.434 INFO [0] client ExtentClose { repair_id: ReconciliationId(425), extent_id: 9 }
Sep 22 23:22:22.434 INFO [1] received reconcile message
Sep 22 23:22:22.434 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(425), op: ExtentClose { repair_id: ReconciliationId(425), extent_id: 9 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.434 INFO [1] client ExtentClose { repair_id: ReconciliationId(425), extent_id: 9 }
Sep 22 23:22:22.434 INFO [2] received reconcile message
Sep 22 23:22:22.434 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(425), op: ExtentClose { repair_id: ReconciliationId(425), extent_id: 9 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.434 INFO [2] client ExtentClose { repair_id: ReconciliationId(425), extent_id: 9 }
Sep 22 23:22:22.434 DEBG 425 Close extent 9
Sep 22 23:22:22.435 DEBG 425 Close extent 9
Sep 22 23:22:22.435 DEBG 425 Close extent 9
Sep 22 23:22:22.435 DEBG [2] It's time to notify for 425
Sep 22 23:22:22.435 INFO Completion from [2] id:425 status:true
Sep 22 23:22:22.435 INFO [426/752] Repair commands completed
Sep 22 23:22:22.435 INFO Pop front: ReconcileIO { id: ReconciliationId(426), op: ExtentRepair { repair_id: ReconciliationId(426), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.435 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.436 INFO [0] received reconcile message
Sep 22 23:22:22.436 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(426), op: ExtentRepair { repair_id: ReconciliationId(426), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.436 INFO [0] client ExtentRepair { repair_id: ReconciliationId(426), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.436 INFO [0] Sending repair request ReconciliationId(426)
Sep 22 23:22:22.436 INFO [1] received reconcile message
Sep 22 23:22:22.436 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(426), op: ExtentRepair { repair_id: ReconciliationId(426), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.436 INFO [1] client ExtentRepair { repair_id: ReconciliationId(426), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.436 INFO [1] No action required ReconciliationId(426)
Sep 22 23:22:22.436 INFO [2] received reconcile message
Sep 22 23:22:22.436 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(426), op: ExtentRepair { repair_id: ReconciliationId(426), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.436 INFO [2] client ExtentRepair { repair_id: ReconciliationId(426), extent_id: 9, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.436 INFO [2] No action required ReconciliationId(426)
Sep 22 23:22:22.436 DEBG 426 Repair extent 9 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.436 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/009.copy"
Sep 22 23:22:22.500 INFO accepted connection, remote_addr: 127.0.0.1:51090, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.501 TRCE incoming request, uri: /extent/9/files, method: GET, req_id: bebd1bf1-1ff9-4206-bc31-69ebb6ff499e, remote_addr: 127.0.0.1:51090, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.501 INFO request completed, latency_us: 208, response_code: 200, uri: /extent/9/files, method: GET, req_id: bebd1bf1-1ff9-4206-bc31-69ebb6ff499e, remote_addr: 127.0.0.1:51090, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.501 INFO eid:9 Found repair files: ["009", "009.db"]
Sep 22 23:22:22.501 TRCE incoming request, uri: /newextent/9/data, method: GET, req_id: b4c4876a-f563-482e-878a-88392de4c9e6, remote_addr: 127.0.0.1:51090, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.502 INFO request completed, latency_us: 315, response_code: 200, uri: /newextent/9/data, method: GET, req_id: b4c4876a-f563-482e-878a-88392de4c9e6, remote_addr: 127.0.0.1:51090, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.506 TRCE incoming request, uri: /newextent/9/db, method: GET, req_id: 510f2e47-4533-468c-86cc-ffeb6be2d09b, remote_addr: 127.0.0.1:51090, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.507 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/9/db, method: GET, req_id: 510f2e47-4533-468c-86cc-ffeb6be2d09b, remote_addr: 127.0.0.1:51090, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.508 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/009.copy" to "/tmp/downstairs-zrMnlo6G/00/000/009.replace"
Sep 22 23:22:22.508 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.509 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/009.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.509 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/009"
Sep 22 23:22:22.509 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/009.db"
Sep 22 23:22:22.509 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.509 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/009.replace" to "/tmp/downstairs-zrMnlo6G/00/000/009.completed"
Sep 22 23:22:22.509 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.509 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.509 DEBG [0] It's time to notify for 426
Sep 22 23:22:22.510 INFO Completion from [0] id:426 status:true
Sep 22 23:22:22.510 INFO [427/752] Repair commands completed
Sep 22 23:22:22.510 INFO Pop front: ReconcileIO { id: ReconciliationId(427), op: ExtentReopen { repair_id: ReconciliationId(427), extent_id: 9 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.510 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.510 INFO [0] received reconcile message
Sep 22 23:22:22.510 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(427), op: ExtentReopen { repair_id: ReconciliationId(427), extent_id: 9 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.510 INFO [0] client ExtentReopen { repair_id: ReconciliationId(427), extent_id: 9 }
Sep 22 23:22:22.510 INFO [1] received reconcile message
Sep 22 23:22:22.510 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(427), op: ExtentReopen { repair_id: ReconciliationId(427), extent_id: 9 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.510 INFO [1] client ExtentReopen { repair_id: ReconciliationId(427), extent_id: 9 }
Sep 22 23:22:22.510 INFO [2] received reconcile message
Sep 22 23:22:22.510 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(427), op: ExtentReopen { repair_id: ReconciliationId(427), extent_id: 9 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.510 INFO [2] client ExtentReopen { repair_id: ReconciliationId(427), extent_id: 9 }
Sep 22 23:22:22.510 DEBG 427 Reopen extent 9
Sep 22 23:22:22.511 DEBG 427 Reopen extent 9
Sep 22 23:22:22.511 DEBG 427 Reopen extent 9
Sep 22 23:22:22.512 DEBG [2] It's time to notify for 427
Sep 22 23:22:22.512 INFO Completion from [2] id:427 status:true
Sep 22 23:22:22.512 INFO [428/752] Repair commands completed
Sep 22 23:22:22.512 INFO Pop front: ReconcileIO { id: ReconciliationId(428), op: ExtentFlush { repair_id: ReconciliationId(428), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.512 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.512 INFO [0] received reconcile message
Sep 22 23:22:22.512 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(428), op: ExtentFlush { repair_id: ReconciliationId(428), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.512 INFO [0] client ExtentFlush { repair_id: ReconciliationId(428), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.512 INFO [1] received reconcile message
Sep 22 23:22:22.512 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(428), op: ExtentFlush { repair_id: ReconciliationId(428), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.512 INFO [1] client ExtentFlush { repair_id: ReconciliationId(428), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.512 INFO [2] received reconcile message
Sep 22 23:22:22.512 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(428), op: ExtentFlush { repair_id: ReconciliationId(428), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.512 INFO [2] client ExtentFlush { repair_id: ReconciliationId(428), extent_id: 186, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.512 DEBG 428 Flush extent 186 with f:2 g:2
Sep 22 23:22:22.512 DEBG Flush just extent 186 with f:2 and g:2
Sep 22 23:22:22.512 DEBG [1] It's time to notify for 428
Sep 22 23:22:22.512 INFO Completion from [1] id:428 status:true
Sep 22 23:22:22.512 INFO [429/752] Repair commands completed
Sep 22 23:22:22.512 INFO Pop front: ReconcileIO { id: ReconciliationId(429), op: ExtentClose { repair_id: ReconciliationId(429), extent_id: 186 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.512 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.512 INFO [0] received reconcile message
Sep 22 23:22:22.513 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(429), op: ExtentClose { repair_id: ReconciliationId(429), extent_id: 186 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.513 INFO [0] client ExtentClose { repair_id: ReconciliationId(429), extent_id: 186 }
Sep 22 23:22:22.513 INFO [1] received reconcile message
Sep 22 23:22:22.513 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(429), op: ExtentClose { repair_id: ReconciliationId(429), extent_id: 186 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.513 INFO [1] client ExtentClose { repair_id: ReconciliationId(429), extent_id: 186 }
Sep 22 23:22:22.513 INFO [2] received reconcile message
Sep 22 23:22:22.513 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(429), op: ExtentClose { repair_id: ReconciliationId(429), extent_id: 186 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.513 INFO [2] client ExtentClose { repair_id: ReconciliationId(429), extent_id: 186 }
Sep 22 23:22:22.513 DEBG 429 Close extent 186
Sep 22 23:22:22.513 DEBG 429 Close extent 186
Sep 22 23:22:22.513 DEBG 429 Close extent 186
Sep 22 23:22:22.514 DEBG [2] It's time to notify for 429
Sep 22 23:22:22.514 INFO Completion from [2] id:429 status:true
Sep 22 23:22:22.514 INFO [430/752] Repair commands completed
Sep 22 23:22:22.514 INFO Pop front: ReconcileIO { id: ReconciliationId(430), op: ExtentRepair { repair_id: ReconciliationId(430), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.514 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.514 INFO [0] received reconcile message
Sep 22 23:22:22.514 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(430), op: ExtentRepair { repair_id: ReconciliationId(430), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.514 INFO [0] client ExtentRepair { repair_id: ReconciliationId(430), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.514 INFO [0] Sending repair request ReconciliationId(430)
Sep 22 23:22:22.514 INFO [1] received reconcile message
Sep 22 23:22:22.514 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(430), op: ExtentRepair { repair_id: ReconciliationId(430), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.514 INFO [1] client ExtentRepair { repair_id: ReconciliationId(430), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.514 INFO [1] No action required ReconciliationId(430)
Sep 22 23:22:22.514 INFO [2] received reconcile message
Sep 22 23:22:22.514 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(430), op: ExtentRepair { repair_id: ReconciliationId(430), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.514 INFO [2] client ExtentRepair { repair_id: ReconciliationId(430), extent_id: 186, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.514 INFO [2] No action required ReconciliationId(430)
Sep 22 23:22:22.514 DEBG 430 Repair extent 186 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.514 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0BA.copy"
Sep 22 23:22:22.553 DEBG [0] Read AckReady 1085, : downstairs
Sep 22 23:22:22.554 DEBG up_ds_listen was notified
Sep 22 23:22:22.555 DEBG up_ds_listen process 1085
Sep 22 23:22:22.555 DEBG [A] ack job 1085:86, : downstairs
Sep 22 23:22:22.579 INFO accepted connection, remote_addr: 127.0.0.1:38910, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.579 TRCE incoming request, uri: /extent/186/files, method: GET, req_id: c8bd42f2-88fa-4bd2-94ff-b44d949588ff, remote_addr: 127.0.0.1:38910, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.579 INFO request completed, latency_us: 202, response_code: 200, uri: /extent/186/files, method: GET, req_id: c8bd42f2-88fa-4bd2-94ff-b44d949588ff, remote_addr: 127.0.0.1:38910, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.579 INFO eid:186 Found repair files: ["0BA", "0BA.db"]
Sep 22 23:22:22.580 TRCE incoming request, uri: /newextent/186/data, method: GET, req_id: 695c9b92-f1b2-46c7-ba20-72a0e9b8f8da, remote_addr: 127.0.0.1:38910, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.580 INFO request completed, latency_us: 261, response_code: 200, uri: /newextent/186/data, method: GET, req_id: 695c9b92-f1b2-46c7-ba20-72a0e9b8f8da, remote_addr: 127.0.0.1:38910, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.585 TRCE incoming request, uri: /newextent/186/db, method: GET, req_id: d5f0695a-18cc-44c8-948c-7d05792d94c1, remote_addr: 127.0.0.1:38910, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.585 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/186/db, method: GET, req_id: d5f0695a-18cc-44c8-948c-7d05792d94c1, remote_addr: 127.0.0.1:38910, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.586 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0BA.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0BA.replace"
Sep 22 23:22:22.586 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.587 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0BA.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.587 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0BA"
Sep 22 23:22:22.587 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0BA.db"
Sep 22 23:22:22.588 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.588 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0BA.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0BA.completed"
Sep 22 23:22:22.588 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.588 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.588 DEBG [0] It's time to notify for 430
Sep 22 23:22:22.588 INFO Completion from [0] id:430 status:true
Sep 22 23:22:22.588 INFO [431/752] Repair commands completed
Sep 22 23:22:22.588 INFO Pop front: ReconcileIO { id: ReconciliationId(431), op: ExtentReopen { repair_id: ReconciliationId(431), extent_id: 186 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.588 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.588 INFO [0] received reconcile message
Sep 22 23:22:22.588 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(431), op: ExtentReopen { repair_id: ReconciliationId(431), extent_id: 186 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.588 INFO [0] client ExtentReopen { repair_id: ReconciliationId(431), extent_id: 186 }
Sep 22 23:22:22.588 INFO [1] received reconcile message
Sep 22 23:22:22.588 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(431), op: ExtentReopen { repair_id: ReconciliationId(431), extent_id: 186 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.588 INFO [1] client ExtentReopen { repair_id: ReconciliationId(431), extent_id: 186 }
Sep 22 23:22:22.588 INFO [2] received reconcile message
Sep 22 23:22:22.588 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(431), op: ExtentReopen { repair_id: ReconciliationId(431), extent_id: 186 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.588 INFO [2] client ExtentReopen { repair_id: ReconciliationId(431), extent_id: 186 }
Sep 22 23:22:22.588 DEBG 431 Reopen extent 186
Sep 22 23:22:22.589 DEBG 431 Reopen extent 186
Sep 22 23:22:22.590 DEBG 431 Reopen extent 186
Sep 22 23:22:22.590 DEBG [2] It's time to notify for 431
Sep 22 23:22:22.590 INFO Completion from [2] id:431 status:true
Sep 22 23:22:22.590 INFO [432/752] Repair commands completed
Sep 22 23:22:22.590 INFO Pop front: ReconcileIO { id: ReconciliationId(432), op: ExtentFlush { repair_id: ReconciliationId(432), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.590 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.590 INFO [0] received reconcile message
Sep 22 23:22:22.590 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(432), op: ExtentFlush { repair_id: ReconciliationId(432), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.590 INFO [0] client ExtentFlush { repair_id: ReconciliationId(432), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.590 INFO [1] received reconcile message
Sep 22 23:22:22.590 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(432), op: ExtentFlush { repair_id: ReconciliationId(432), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.590 INFO [1] client ExtentFlush { repair_id: ReconciliationId(432), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.590 INFO [2] received reconcile message
Sep 22 23:22:22.590 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(432), op: ExtentFlush { repair_id: ReconciliationId(432), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.590 INFO [2] client ExtentFlush { repair_id: ReconciliationId(432), extent_id: 130, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.591 DEBG 432 Flush extent 130 with f:2 g:2
Sep 22 23:22:22.591 DEBG Flush just extent 130 with f:2 and g:2
Sep 22 23:22:22.591 DEBG [1] It's time to notify for 432
Sep 22 23:22:22.591 INFO Completion from [1] id:432 status:true
Sep 22 23:22:22.591 INFO [433/752] Repair commands completed
Sep 22 23:22:22.591 INFO Pop front: ReconcileIO { id: ReconciliationId(433), op: ExtentClose { repair_id: ReconciliationId(433), extent_id: 130 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.591 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.591 INFO [0] received reconcile message
Sep 22 23:22:22.591 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(433), op: ExtentClose { repair_id: ReconciliationId(433), extent_id: 130 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.591 INFO [0] client ExtentClose { repair_id: ReconciliationId(433), extent_id: 130 }
Sep 22 23:22:22.591 INFO [1] received reconcile message
Sep 22 23:22:22.591 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(433), op: ExtentClose { repair_id: ReconciliationId(433), extent_id: 130 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.591 INFO [1] client ExtentClose { repair_id: ReconciliationId(433), extent_id: 130 }
Sep 22 23:22:22.591 INFO [2] received reconcile message
Sep 22 23:22:22.591 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(433), op: ExtentClose { repair_id: ReconciliationId(433), extent_id: 130 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.591 INFO [2] client ExtentClose { repair_id: ReconciliationId(433), extent_id: 130 }
Sep 22 23:22:22.591 DEBG 433 Close extent 130
Sep 22 23:22:22.591 DEBG 433 Close extent 130
Sep 22 23:22:22.592 DEBG 433 Close extent 130
Sep 22 23:22:22.592 DEBG [2] It's time to notify for 433
Sep 22 23:22:22.592 INFO Completion from [2] id:433 status:true
Sep 22 23:22:22.592 INFO [434/752] Repair commands completed
Sep 22 23:22:22.592 INFO Pop front: ReconcileIO { id: ReconciliationId(434), op: ExtentRepair { repair_id: ReconciliationId(434), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.592 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.592 INFO [0] received reconcile message
Sep 22 23:22:22.592 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(434), op: ExtentRepair { repair_id: ReconciliationId(434), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.592 INFO [0] client ExtentRepair { repair_id: ReconciliationId(434), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.592 INFO [0] Sending repair request ReconciliationId(434)
Sep 22 23:22:22.592 INFO [1] received reconcile message
Sep 22 23:22:22.592 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(434), op: ExtentRepair { repair_id: ReconciliationId(434), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.592 INFO [1] client ExtentRepair { repair_id: ReconciliationId(434), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.592 INFO [1] No action required ReconciliationId(434)
Sep 22 23:22:22.592 INFO [2] received reconcile message
Sep 22 23:22:22.593 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(434), op: ExtentRepair { repair_id: ReconciliationId(434), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.593 INFO [2] client ExtentRepair { repair_id: ReconciliationId(434), extent_id: 130, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.593 INFO [2] No action required ReconciliationId(434)
Sep 22 23:22:22.593 DEBG 434 Repair extent 130 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.593 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/082.copy"
Sep 22 23:22:22.608 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:22.610 INFO [lossy] skipping 1084
Sep 22 23:22:22.610 WARN returning error on flush!
Sep 22 23:22:22.610 DEBG Flush     :1084 extent_limit None deps:[JobId(1083), JobId(1082)] res:false f:31 g:1
Sep 22 23:22:22.610 DEBG Flush     :1084 extent_limit None deps:[JobId(1083), JobId(1082)] res:true f:31 g:1
Sep 22 23:22:22.610 INFO [lossy] sleeping 1 second
Sep 22 23:22:22.610 DEBG IO Flush 1086 has deps [JobId(1085), JobId(1084)]
Sep 22 23:22:22.611 WARN returning error on read!
Sep 22 23:22:22.611 DEBG Read      :1085 deps:[JobId(1084)] res:false
Sep 22 23:22:22.616 DEBG Read      :1085 deps:[JobId(1084)] res:true
Sep 22 23:22:22.655 INFO accepted connection, remote_addr: 127.0.0.1:51399, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.655 TRCE incoming request, uri: /extent/130/files, method: GET, req_id: 5f59ca70-05a0-4cbb-9a3b-f15103f9d530, remote_addr: 127.0.0.1:51399, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.655 INFO request completed, latency_us: 209, response_code: 200, uri: /extent/130/files, method: GET, req_id: 5f59ca70-05a0-4cbb-9a3b-f15103f9d530, remote_addr: 127.0.0.1:51399, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.656 INFO eid:130 Found repair files: ["082", "082.db"]
Sep 22 23:22:22.656 TRCE incoming request, uri: /newextent/130/data, method: GET, req_id: 394cb65e-e60c-4787-88aa-76b22081db71, remote_addr: 127.0.0.1:51399, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.656 INFO request completed, latency_us: 333, response_code: 200, uri: /newextent/130/data, method: GET, req_id: 394cb65e-e60c-4787-88aa-76b22081db71, remote_addr: 127.0.0.1:51399, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.661 TRCE incoming request, uri: /newextent/130/db, method: GET, req_id: 5585788a-dd13-4f6a-9cfb-eebd32258be1, remote_addr: 127.0.0.1:51399, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.661 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/130/db, method: GET, req_id: 5585788a-dd13-4f6a-9cfb-eebd32258be1, remote_addr: 127.0.0.1:51399, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.662 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/082.copy" to "/tmp/downstairs-zrMnlo6G/00/000/082.replace"
Sep 22 23:22:22.662 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.663 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/082.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.663 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/082"
Sep 22 23:22:22.663 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/082.db"
Sep 22 23:22:22.663 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.663 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/082.replace" to "/tmp/downstairs-zrMnlo6G/00/000/082.completed"
Sep 22 23:22:22.664 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.664 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.664 DEBG [0] It's time to notify for 434
Sep 22 23:22:22.664 INFO Completion from [0] id:434 status:true
Sep 22 23:22:22.664 INFO [435/752] Repair commands completed
Sep 22 23:22:22.664 INFO Pop front: ReconcileIO { id: ReconciliationId(435), op: ExtentReopen { repair_id: ReconciliationId(435), extent_id: 130 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.664 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.664 INFO [0] received reconcile message
Sep 22 23:22:22.664 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(435), op: ExtentReopen { repair_id: ReconciliationId(435), extent_id: 130 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.664 INFO [0] client ExtentReopen { repair_id: ReconciliationId(435), extent_id: 130 }
Sep 22 23:22:22.664 INFO [1] received reconcile message
Sep 22 23:22:22.664 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(435), op: ExtentReopen { repair_id: ReconciliationId(435), extent_id: 130 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.664 INFO [1] client ExtentReopen { repair_id: ReconciliationId(435), extent_id: 130 }
Sep 22 23:22:22.664 INFO [2] received reconcile message
Sep 22 23:22:22.664 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(435), op: ExtentReopen { repair_id: ReconciliationId(435), extent_id: 130 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.664 INFO [2] client ExtentReopen { repair_id: ReconciliationId(435), extent_id: 130 }
Sep 22 23:22:22.664 DEBG 435 Reopen extent 130
Sep 22 23:22:22.665 DEBG 435 Reopen extent 130
Sep 22 23:22:22.666 DEBG 435 Reopen extent 130
Sep 22 23:22:22.666 DEBG [2] It's time to notify for 435
Sep 22 23:22:22.666 INFO Completion from [2] id:435 status:true
Sep 22 23:22:22.666 INFO [436/752] Repair commands completed
Sep 22 23:22:22.666 INFO Pop front: ReconcileIO { id: ReconciliationId(436), op: ExtentFlush { repair_id: ReconciliationId(436), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.666 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.666 INFO [0] received reconcile message
Sep 22 23:22:22.666 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(436), op: ExtentFlush { repair_id: ReconciliationId(436), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.666 INFO [0] client ExtentFlush { repair_id: ReconciliationId(436), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.666 INFO [1] received reconcile message
Sep 22 23:22:22.666 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(436), op: ExtentFlush { repair_id: ReconciliationId(436), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.666 INFO [1] client ExtentFlush { repair_id: ReconciliationId(436), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.666 INFO [2] received reconcile message
Sep 22 23:22:22.666 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(436), op: ExtentFlush { repair_id: ReconciliationId(436), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.666 INFO [2] client ExtentFlush { repair_id: ReconciliationId(436), extent_id: 122, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.667 DEBG 436 Flush extent 122 with f:2 g:2
Sep 22 23:22:22.667 DEBG Flush just extent 122 with f:2 and g:2
Sep 22 23:22:22.667 DEBG [1] It's time to notify for 436
Sep 22 23:22:22.667 INFO Completion from [1] id:436 status:true
Sep 22 23:22:22.667 INFO [437/752] Repair commands completed
Sep 22 23:22:22.667 INFO Pop front: ReconcileIO { id: ReconciliationId(437), op: ExtentClose { repair_id: ReconciliationId(437), extent_id: 122 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.667 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.667 INFO [0] received reconcile message
Sep 22 23:22:22.667 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(437), op: ExtentClose { repair_id: ReconciliationId(437), extent_id: 122 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.667 INFO [0] client ExtentClose { repair_id: ReconciliationId(437), extent_id: 122 }
Sep 22 23:22:22.667 INFO [1] received reconcile message
Sep 22 23:22:22.667 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(437), op: ExtentClose { repair_id: ReconciliationId(437), extent_id: 122 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.667 INFO [1] client ExtentClose { repair_id: ReconciliationId(437), extent_id: 122 }
Sep 22 23:22:22.667 INFO [2] received reconcile message
Sep 22 23:22:22.667 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(437), op: ExtentClose { repair_id: ReconciliationId(437), extent_id: 122 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.667 INFO [2] client ExtentClose { repair_id: ReconciliationId(437), extent_id: 122 }
Sep 22 23:22:22.667 DEBG 437 Close extent 122
Sep 22 23:22:22.667 DEBG 437 Close extent 122
Sep 22 23:22:22.668 DEBG 437 Close extent 122
Sep 22 23:22:22.668 DEBG [2] It's time to notify for 437
Sep 22 23:22:22.668 INFO Completion from [2] id:437 status:true
Sep 22 23:22:22.668 INFO [438/752] Repair commands completed
Sep 22 23:22:22.668 INFO Pop front: ReconcileIO { id: ReconciliationId(438), op: ExtentRepair { repair_id: ReconciliationId(438), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.668 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.668 INFO [0] received reconcile message
Sep 22 23:22:22.668 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(438), op: ExtentRepair { repair_id: ReconciliationId(438), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.668 INFO [0] client ExtentRepair { repair_id: ReconciliationId(438), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.668 INFO [0] Sending repair request ReconciliationId(438)
Sep 22 23:22:22.668 INFO [1] received reconcile message
Sep 22 23:22:22.668 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(438), op: ExtentRepair { repair_id: ReconciliationId(438), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.668 INFO [1] client ExtentRepair { repair_id: ReconciliationId(438), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.668 INFO [1] No action required ReconciliationId(438)
Sep 22 23:22:22.669 INFO [2] received reconcile message
Sep 22 23:22:22.669 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(438), op: ExtentRepair { repair_id: ReconciliationId(438), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.669 INFO [2] client ExtentRepair { repair_id: ReconciliationId(438), extent_id: 122, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.669 INFO [2] No action required ReconciliationId(438)
Sep 22 23:22:22.669 DEBG 438 Repair extent 122 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.669 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/07A.copy"
Sep 22 23:22:22.686 DEBG IO Read  1087 has deps [JobId(1086)]
Sep 22 23:22:22.686 ERRO [1] job id 1085 saw error GenericError("test error")
Sep 22 23:22:22.731 INFO accepted connection, remote_addr: 127.0.0.1:48075, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.731 TRCE incoming request, uri: /extent/122/files, method: GET, req_id: 92ca1dce-fde1-4120-931d-837f7da5c7fe, remote_addr: 127.0.0.1:48075, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.731 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/122/files, method: GET, req_id: 92ca1dce-fde1-4120-931d-837f7da5c7fe, remote_addr: 127.0.0.1:48075, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.732 INFO eid:122 Found repair files: ["07A", "07A.db"]
Sep 22 23:22:22.732 TRCE incoming request, uri: /newextent/122/data, method: GET, req_id: 8ea21843-fa3d-45c4-bff7-d8881f7b135b, remote_addr: 127.0.0.1:48075, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.732 INFO request completed, latency_us: 324, response_code: 200, uri: /newextent/122/data, method: GET, req_id: 8ea21843-fa3d-45c4-bff7-d8881f7b135b, remote_addr: 127.0.0.1:48075, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.737 TRCE incoming request, uri: /newextent/122/db, method: GET, req_id: b1134cf7-f68a-48b2-aa26-a38a22d21d1d, remote_addr: 127.0.0.1:48075, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.737 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/122/db, method: GET, req_id: b1134cf7-f68a-48b2-aa26-a38a22d21d1d, remote_addr: 127.0.0.1:48075, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.738 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/07A.copy" to "/tmp/downstairs-zrMnlo6G/00/000/07A.replace"
Sep 22 23:22:22.738 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.739 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/07A.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.739 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07A"
Sep 22 23:22:22.740 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/07A.db"
Sep 22 23:22:22.740 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.740 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/07A.replace" to "/tmp/downstairs-zrMnlo6G/00/000/07A.completed"
Sep 22 23:22:22.740 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.740 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.740 DEBG [0] It's time to notify for 438
Sep 22 23:22:22.740 INFO Completion from [0] id:438 status:true
Sep 22 23:22:22.740 INFO [439/752] Repair commands completed
Sep 22 23:22:22.740 INFO Pop front: ReconcileIO { id: ReconciliationId(439), op: ExtentReopen { repair_id: ReconciliationId(439), extent_id: 122 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.740 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.740 INFO [0] received reconcile message
Sep 22 23:22:22.740 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(439), op: ExtentReopen { repair_id: ReconciliationId(439), extent_id: 122 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.740 INFO [0] client ExtentReopen { repair_id: ReconciliationId(439), extent_id: 122 }
Sep 22 23:22:22.740 INFO [1] received reconcile message
Sep 22 23:22:22.740 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(439), op: ExtentReopen { repair_id: ReconciliationId(439), extent_id: 122 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.740 INFO [1] client ExtentReopen { repair_id: ReconciliationId(439), extent_id: 122 }
Sep 22 23:22:22.740 INFO [2] received reconcile message
Sep 22 23:22:22.740 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(439), op: ExtentReopen { repair_id: ReconciliationId(439), extent_id: 122 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.740 INFO [2] client ExtentReopen { repair_id: ReconciliationId(439), extent_id: 122 }
Sep 22 23:22:22.740 DEBG 439 Reopen extent 122
Sep 22 23:22:22.741 DEBG 439 Reopen extent 122
Sep 22 23:22:22.742 DEBG 439 Reopen extent 122
Sep 22 23:22:22.742 DEBG [2] It's time to notify for 439
Sep 22 23:22:22.742 INFO Completion from [2] id:439 status:true
Sep 22 23:22:22.742 INFO [440/752] Repair commands completed
Sep 22 23:22:22.742 INFO Pop front: ReconcileIO { id: ReconciliationId(440), op: ExtentFlush { repair_id: ReconciliationId(440), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.742 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.742 INFO [0] received reconcile message
Sep 22 23:22:22.742 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(440), op: ExtentFlush { repair_id: ReconciliationId(440), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.742 INFO [0] client ExtentFlush { repair_id: ReconciliationId(440), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.742 INFO [1] received reconcile message
Sep 22 23:22:22.742 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(440), op: ExtentFlush { repair_id: ReconciliationId(440), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.742 INFO [1] client ExtentFlush { repair_id: ReconciliationId(440), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.742 INFO [2] received reconcile message
Sep 22 23:22:22.742 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(440), op: ExtentFlush { repair_id: ReconciliationId(440), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.742 INFO [2] client ExtentFlush { repair_id: ReconciliationId(440), extent_id: 105, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.743 DEBG 440 Flush extent 105 with f:2 g:2
Sep 22 23:22:22.743 DEBG Flush just extent 105 with f:2 and g:2
Sep 22 23:22:22.743 DEBG [1] It's time to notify for 440
Sep 22 23:22:22.743 INFO Completion from [1] id:440 status:true
Sep 22 23:22:22.743 INFO [441/752] Repair commands completed
Sep 22 23:22:22.743 INFO Pop front: ReconcileIO { id: ReconciliationId(441), op: ExtentClose { repair_id: ReconciliationId(441), extent_id: 105 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.743 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.743 INFO [0] received reconcile message
Sep 22 23:22:22.743 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(441), op: ExtentClose { repair_id: ReconciliationId(441), extent_id: 105 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.743 INFO [0] client ExtentClose { repair_id: ReconciliationId(441), extent_id: 105 }
Sep 22 23:22:22.743 INFO [1] received reconcile message
Sep 22 23:22:22.743 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(441), op: ExtentClose { repair_id: ReconciliationId(441), extent_id: 105 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.743 INFO [1] client ExtentClose { repair_id: ReconciliationId(441), extent_id: 105 }
Sep 22 23:22:22.743 INFO [2] received reconcile message
Sep 22 23:22:22.743 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(441), op: ExtentClose { repair_id: ReconciliationId(441), extent_id: 105 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.743 INFO [2] client ExtentClose { repair_id: ReconciliationId(441), extent_id: 105 }
Sep 22 23:22:22.743 DEBG 441 Close extent 105
Sep 22 23:22:22.743 DEBG 441 Close extent 105
Sep 22 23:22:22.744 DEBG 441 Close extent 105
Sep 22 23:22:22.744 DEBG [2] It's time to notify for 441
Sep 22 23:22:22.744 INFO Completion from [2] id:441 status:true
Sep 22 23:22:22.744 INFO [442/752] Repair commands completed
Sep 22 23:22:22.744 INFO Pop front: ReconcileIO { id: ReconciliationId(442), op: ExtentRepair { repair_id: ReconciliationId(442), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.744 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.744 INFO [0] received reconcile message
Sep 22 23:22:22.744 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(442), op: ExtentRepair { repair_id: ReconciliationId(442), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.744 INFO [0] client ExtentRepair { repair_id: ReconciliationId(442), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.744 INFO [0] Sending repair request ReconciliationId(442)
Sep 22 23:22:22.744 INFO [1] received reconcile message
Sep 22 23:22:22.744 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(442), op: ExtentRepair { repair_id: ReconciliationId(442), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.744 INFO [1] client ExtentRepair { repair_id: ReconciliationId(442), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.744 INFO [1] No action required ReconciliationId(442)
Sep 22 23:22:22.745 INFO [2] received reconcile message
Sep 22 23:22:22.745 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(442), op: ExtentRepair { repair_id: ReconciliationId(442), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.745 INFO [2] client ExtentRepair { repair_id: ReconciliationId(442), extent_id: 105, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.745 INFO [2] No action required ReconciliationId(442)
Sep 22 23:22:22.745 DEBG 442 Repair extent 105 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.745 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/069.copy"
Sep 22 23:22:22.810 INFO accepted connection, remote_addr: 127.0.0.1:36700, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.810 TRCE incoming request, uri: /extent/105/files, method: GET, req_id: 15ff1093-a55b-470a-bfda-1120ec4c7ac2, remote_addr: 127.0.0.1:36700, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.811 INFO request completed, latency_us: 204, response_code: 200, uri: /extent/105/files, method: GET, req_id: 15ff1093-a55b-470a-bfda-1120ec4c7ac2, remote_addr: 127.0.0.1:36700, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.811 INFO eid:105 Found repair files: ["069", "069.db"]
Sep 22 23:22:22.811 TRCE incoming request, uri: /newextent/105/data, method: GET, req_id: dee41f68-0c16-4c5b-8af8-38c73a0ff2bf, remote_addr: 127.0.0.1:36700, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.811 INFO request completed, latency_us: 328, response_code: 200, uri: /newextent/105/data, method: GET, req_id: dee41f68-0c16-4c5b-8af8-38c73a0ff2bf, remote_addr: 127.0.0.1:36700, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.816 TRCE incoming request, uri: /newextent/105/db, method: GET, req_id: 4920d333-733a-4a77-ae57-7c1d85db3686, remote_addr: 127.0.0.1:36700, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.816 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/105/db, method: GET, req_id: 4920d333-733a-4a77-ae57-7c1d85db3686, remote_addr: 127.0.0.1:36700, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.818 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/069.copy" to "/tmp/downstairs-zrMnlo6G/00/000/069.replace"
Sep 22 23:22:22.818 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.818 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/069.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.819 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/069"
Sep 22 23:22:22.819 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/069.db"
Sep 22 23:22:22.819 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.819 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/069.replace" to "/tmp/downstairs-zrMnlo6G/00/000/069.completed"
Sep 22 23:22:22.819 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.819 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.819 DEBG [0] It's time to notify for 442
Sep 22 23:22:22.819 INFO Completion from [0] id:442 status:true
Sep 22 23:22:22.819 INFO [443/752] Repair commands completed
Sep 22 23:22:22.819 INFO Pop front: ReconcileIO { id: ReconciliationId(443), op: ExtentReopen { repair_id: ReconciliationId(443), extent_id: 105 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.819 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.819 INFO [0] received reconcile message
Sep 22 23:22:22.819 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(443), op: ExtentReopen { repair_id: ReconciliationId(443), extent_id: 105 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.819 INFO [0] client ExtentReopen { repair_id: ReconciliationId(443), extent_id: 105 }
Sep 22 23:22:22.819 INFO [1] received reconcile message
Sep 22 23:22:22.819 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(443), op: ExtentReopen { repair_id: ReconciliationId(443), extent_id: 105 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.819 INFO [1] client ExtentReopen { repair_id: ReconciliationId(443), extent_id: 105 }
Sep 22 23:22:22.819 INFO [2] received reconcile message
Sep 22 23:22:22.819 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(443), op: ExtentReopen { repair_id: ReconciliationId(443), extent_id: 105 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.819 INFO [2] client ExtentReopen { repair_id: ReconciliationId(443), extent_id: 105 }
Sep 22 23:22:22.820 DEBG 443 Reopen extent 105
Sep 22 23:22:22.820 DEBG 443 Reopen extent 105
Sep 22 23:22:22.821 DEBG 443 Reopen extent 105
Sep 22 23:22:22.821 DEBG [2] It's time to notify for 443
Sep 22 23:22:22.821 INFO Completion from [2] id:443 status:true
Sep 22 23:22:22.821 INFO [444/752] Repair commands completed
Sep 22 23:22:22.821 INFO Pop front: ReconcileIO { id: ReconciliationId(444), op: ExtentFlush { repair_id: ReconciliationId(444), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.821 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.821 INFO [0] received reconcile message
Sep 22 23:22:22.821 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(444), op: ExtentFlush { repair_id: ReconciliationId(444), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.822 INFO [0] client ExtentFlush { repair_id: ReconciliationId(444), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.822 INFO [1] received reconcile message
Sep 22 23:22:22.822 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(444), op: ExtentFlush { repair_id: ReconciliationId(444), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.822 INFO [1] client ExtentFlush { repair_id: ReconciliationId(444), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.822 INFO [2] received reconcile message
Sep 22 23:22:22.822 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(444), op: ExtentFlush { repair_id: ReconciliationId(444), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.822 INFO [2] client ExtentFlush { repair_id: ReconciliationId(444), extent_id: 137, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.822 DEBG 444 Flush extent 137 with f:2 g:2
Sep 22 23:22:22.822 DEBG Flush just extent 137 with f:2 and g:2
Sep 22 23:22:22.822 DEBG [1] It's time to notify for 444
Sep 22 23:22:22.822 INFO Completion from [1] id:444 status:true
Sep 22 23:22:22.822 INFO [445/752] Repair commands completed
Sep 22 23:22:22.822 INFO Pop front: ReconcileIO { id: ReconciliationId(445), op: ExtentClose { repair_id: ReconciliationId(445), extent_id: 137 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.822 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.822 INFO [0] received reconcile message
Sep 22 23:22:22.822 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(445), op: ExtentClose { repair_id: ReconciliationId(445), extent_id: 137 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.822 INFO [0] client ExtentClose { repair_id: ReconciliationId(445), extent_id: 137 }
Sep 22 23:22:22.822 INFO [1] received reconcile message
Sep 22 23:22:22.822 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(445), op: ExtentClose { repair_id: ReconciliationId(445), extent_id: 137 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.822 INFO [1] client ExtentClose { repair_id: ReconciliationId(445), extent_id: 137 }
Sep 22 23:22:22.822 INFO [2] received reconcile message
Sep 22 23:22:22.822 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(445), op: ExtentClose { repair_id: ReconciliationId(445), extent_id: 137 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.822 INFO [2] client ExtentClose { repair_id: ReconciliationId(445), extent_id: 137 }
Sep 22 23:22:22.822 DEBG 445 Close extent 137
Sep 22 23:22:22.823 DEBG 445 Close extent 137
Sep 22 23:22:22.823 DEBG 445 Close extent 137
Sep 22 23:22:22.823 DEBG [2] It's time to notify for 445
Sep 22 23:22:22.823 INFO Completion from [2] id:445 status:true
Sep 22 23:22:22.823 INFO [446/752] Repair commands completed
Sep 22 23:22:22.823 INFO Pop front: ReconcileIO { id: ReconciliationId(446), op: ExtentRepair { repair_id: ReconciliationId(446), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.823 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.823 INFO [0] received reconcile message
Sep 22 23:22:22.824 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(446), op: ExtentRepair { repair_id: ReconciliationId(446), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.824 INFO [0] client ExtentRepair { repair_id: ReconciliationId(446), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.824 INFO [0] Sending repair request ReconciliationId(446)
Sep 22 23:22:22.824 INFO [1] received reconcile message
Sep 22 23:22:22.824 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(446), op: ExtentRepair { repair_id: ReconciliationId(446), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.824 INFO [1] client ExtentRepair { repair_id: ReconciliationId(446), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.824 INFO [1] No action required ReconciliationId(446)
Sep 22 23:22:22.824 INFO [2] received reconcile message
Sep 22 23:22:22.824 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(446), op: ExtentRepair { repair_id: ReconciliationId(446), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.824 INFO [2] client ExtentRepair { repair_id: ReconciliationId(446), extent_id: 137, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.824 INFO [2] No action required ReconciliationId(446)
Sep 22 23:22:22.824 DEBG 446 Repair extent 137 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.824 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/089.copy"
Sep 22 23:22:22.886 INFO accepted connection, remote_addr: 127.0.0.1:44079, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.886 TRCE incoming request, uri: /extent/137/files, method: GET, req_id: 5ff75bb0-2da9-4dd4-ab22-671ed4cc4804, remote_addr: 127.0.0.1:44079, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.886 INFO request completed, latency_us: 197, response_code: 200, uri: /extent/137/files, method: GET, req_id: 5ff75bb0-2da9-4dd4-ab22-671ed4cc4804, remote_addr: 127.0.0.1:44079, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.887 INFO eid:137 Found repair files: ["089", "089.db"]
Sep 22 23:22:22.887 TRCE incoming request, uri: /newextent/137/data, method: GET, req_id: 5f9407db-c4d9-4a96-b20c-d1b9f022cc0a, remote_addr: 127.0.0.1:44079, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.887 INFO request completed, latency_us: 319, response_code: 200, uri: /newextent/137/data, method: GET, req_id: 5f9407db-c4d9-4a96-b20c-d1b9f022cc0a, remote_addr: 127.0.0.1:44079, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.892 TRCE incoming request, uri: /newextent/137/db, method: GET, req_id: 7e2063e9-d11e-47db-9a7e-7582000e706b, remote_addr: 127.0.0.1:44079, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.892 INFO request completed, latency_us: 305, response_code: 200, uri: /newextent/137/db, method: GET, req_id: 7e2063e9-d11e-47db-9a7e-7582000e706b, remote_addr: 127.0.0.1:44079, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.893 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/089.copy" to "/tmp/downstairs-zrMnlo6G/00/000/089.replace"
Sep 22 23:22:22.893 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.894 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/089.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.894 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/089"
Sep 22 23:22:22.895 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/089.db"
Sep 22 23:22:22.895 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.895 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/089.replace" to "/tmp/downstairs-zrMnlo6G/00/000/089.completed"
Sep 22 23:22:22.895 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.895 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.895 DEBG [0] It's time to notify for 446
Sep 22 23:22:22.895 INFO Completion from [0] id:446 status:true
Sep 22 23:22:22.895 INFO [447/752] Repair commands completed
Sep 22 23:22:22.895 INFO Pop front: ReconcileIO { id: ReconciliationId(447), op: ExtentReopen { repair_id: ReconciliationId(447), extent_id: 137 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.895 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.895 INFO [0] received reconcile message
Sep 22 23:22:22.895 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(447), op: ExtentReopen { repair_id: ReconciliationId(447), extent_id: 137 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.895 INFO [0] client ExtentReopen { repair_id: ReconciliationId(447), extent_id: 137 }
Sep 22 23:22:22.895 INFO [1] received reconcile message
Sep 22 23:22:22.895 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(447), op: ExtentReopen { repair_id: ReconciliationId(447), extent_id: 137 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.895 INFO [1] client ExtentReopen { repair_id: ReconciliationId(447), extent_id: 137 }
Sep 22 23:22:22.895 INFO [2] received reconcile message
Sep 22 23:22:22.895 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(447), op: ExtentReopen { repair_id: ReconciliationId(447), extent_id: 137 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.895 INFO [2] client ExtentReopen { repair_id: ReconciliationId(447), extent_id: 137 }
Sep 22 23:22:22.895 DEBG 447 Reopen extent 137
Sep 22 23:22:22.896 DEBG 447 Reopen extent 137
Sep 22 23:22:22.896 DEBG 447 Reopen extent 137
Sep 22 23:22:22.897 DEBG [2] It's time to notify for 447
Sep 22 23:22:22.897 INFO Completion from [2] id:447 status:true
Sep 22 23:22:22.897 INFO [448/752] Repair commands completed
Sep 22 23:22:22.897 INFO Pop front: ReconcileIO { id: ReconciliationId(448), op: ExtentFlush { repair_id: ReconciliationId(448), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.897 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.897 INFO [0] received reconcile message
Sep 22 23:22:22.897 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(448), op: ExtentFlush { repair_id: ReconciliationId(448), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.897 INFO [0] client ExtentFlush { repair_id: ReconciliationId(448), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.897 INFO [1] received reconcile message
Sep 22 23:22:22.897 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(448), op: ExtentFlush { repair_id: ReconciliationId(448), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.897 INFO [1] client ExtentFlush { repair_id: ReconciliationId(448), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.897 INFO [2] received reconcile message
Sep 22 23:22:22.897 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(448), op: ExtentFlush { repair_id: ReconciliationId(448), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.897 INFO [2] client ExtentFlush { repair_id: ReconciliationId(448), extent_id: 7, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.898 DEBG 448 Flush extent 7 with f:2 g:2
Sep 22 23:22:22.898 DEBG Flush just extent 7 with f:2 and g:2
Sep 22 23:22:22.898 DEBG [1] It's time to notify for 448
Sep 22 23:22:22.898 INFO Completion from [1] id:448 status:true
Sep 22 23:22:22.898 INFO [449/752] Repair commands completed
Sep 22 23:22:22.898 INFO Pop front: ReconcileIO { id: ReconciliationId(449), op: ExtentClose { repair_id: ReconciliationId(449), extent_id: 7 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.898 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.898 INFO [0] received reconcile message
Sep 22 23:22:22.898 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(449), op: ExtentClose { repair_id: ReconciliationId(449), extent_id: 7 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.898 INFO [0] client ExtentClose { repair_id: ReconciliationId(449), extent_id: 7 }
Sep 22 23:22:22.898 INFO [1] received reconcile message
Sep 22 23:22:22.898 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(449), op: ExtentClose { repair_id: ReconciliationId(449), extent_id: 7 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.898 INFO [1] client ExtentClose { repair_id: ReconciliationId(449), extent_id: 7 }
Sep 22 23:22:22.898 INFO [2] received reconcile message
Sep 22 23:22:22.898 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(449), op: ExtentClose { repair_id: ReconciliationId(449), extent_id: 7 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.898 INFO [2] client ExtentClose { repair_id: ReconciliationId(449), extent_id: 7 }
Sep 22 23:22:22.898 DEBG 449 Close extent 7
Sep 22 23:22:22.898 DEBG 449 Close extent 7
Sep 22 23:22:22.899 DEBG 449 Close extent 7
Sep 22 23:22:22.899 DEBG [2] It's time to notify for 449
Sep 22 23:22:22.899 INFO Completion from [2] id:449 status:true
Sep 22 23:22:22.899 INFO [450/752] Repair commands completed
Sep 22 23:22:22.899 INFO Pop front: ReconcileIO { id: ReconciliationId(450), op: ExtentRepair { repair_id: ReconciliationId(450), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.899 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.899 INFO [0] received reconcile message
Sep 22 23:22:22.899 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(450), op: ExtentRepair { repair_id: ReconciliationId(450), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.899 INFO [0] client ExtentRepair { repair_id: ReconciliationId(450), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.899 INFO [0] Sending repair request ReconciliationId(450)
Sep 22 23:22:22.899 INFO [1] received reconcile message
Sep 22 23:22:22.899 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(450), op: ExtentRepair { repair_id: ReconciliationId(450), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.899 INFO [1] client ExtentRepair { repair_id: ReconciliationId(450), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.899 INFO [1] No action required ReconciliationId(450)
Sep 22 23:22:22.899 INFO [2] received reconcile message
Sep 22 23:22:22.899 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(450), op: ExtentRepair { repair_id: ReconciliationId(450), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.899 INFO [2] client ExtentRepair { repair_id: ReconciliationId(450), extent_id: 7, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.900 INFO [2] No action required ReconciliationId(450)
Sep 22 23:22:22.900 DEBG 450 Repair extent 7 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.900 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/007.copy"
Sep 22 23:22:22.962 INFO accepted connection, remote_addr: 127.0.0.1:59962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.962 TRCE incoming request, uri: /extent/7/files, method: GET, req_id: 81d13e72-f213-451c-990d-4b8e63611f7d, remote_addr: 127.0.0.1:59962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.962 INFO request completed, latency_us: 196, response_code: 200, uri: /extent/7/files, method: GET, req_id: 81d13e72-f213-451c-990d-4b8e63611f7d, remote_addr: 127.0.0.1:59962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.962 INFO eid:7 Found repair files: ["007", "007.db"]
Sep 22 23:22:22.963 TRCE incoming request, uri: /newextent/7/data, method: GET, req_id: 8f4984ae-a373-49dc-9f1f-1847cab5d6bc, remote_addr: 127.0.0.1:59962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.963 INFO request completed, latency_us: 259, response_code: 200, uri: /newextent/7/data, method: GET, req_id: 8f4984ae-a373-49dc-9f1f-1847cab5d6bc, remote_addr: 127.0.0.1:59962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.968 TRCE incoming request, uri: /newextent/7/db, method: GET, req_id: b85a5600-8937-458d-abef-5654c0823fbd, remote_addr: 127.0.0.1:59962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.968 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/7/db, method: GET, req_id: b85a5600-8937-458d-abef-5654c0823fbd, remote_addr: 127.0.0.1:59962, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:22.969 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/007.copy" to "/tmp/downstairs-zrMnlo6G/00/000/007.replace"
Sep 22 23:22:22.969 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.970 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/007.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.970 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/007"
Sep 22 23:22:22.970 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/007.db"
Sep 22 23:22:22.970 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.970 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/007.replace" to "/tmp/downstairs-zrMnlo6G/00/000/007.completed"
Sep 22 23:22:22.970 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.970 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:22.971 DEBG [0] It's time to notify for 450
Sep 22 23:22:22.971 INFO Completion from [0] id:450 status:true
Sep 22 23:22:22.971 INFO [451/752] Repair commands completed
Sep 22 23:22:22.971 INFO Pop front: ReconcileIO { id: ReconciliationId(451), op: ExtentReopen { repair_id: ReconciliationId(451), extent_id: 7 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.971 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.971 INFO [0] received reconcile message
Sep 22 23:22:22.971 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(451), op: ExtentReopen { repair_id: ReconciliationId(451), extent_id: 7 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.971 INFO [0] client ExtentReopen { repair_id: ReconciliationId(451), extent_id: 7 }
Sep 22 23:22:22.971 INFO [1] received reconcile message
Sep 22 23:22:22.971 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(451), op: ExtentReopen { repair_id: ReconciliationId(451), extent_id: 7 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.971 INFO [1] client ExtentReopen { repair_id: ReconciliationId(451), extent_id: 7 }
Sep 22 23:22:22.971 INFO [2] received reconcile message
Sep 22 23:22:22.971 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(451), op: ExtentReopen { repair_id: ReconciliationId(451), extent_id: 7 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.971 INFO [2] client ExtentReopen { repair_id: ReconciliationId(451), extent_id: 7 }
Sep 22 23:22:22.971 DEBG 451 Reopen extent 7
Sep 22 23:22:22.972 DEBG 451 Reopen extent 7
Sep 22 23:22:22.972 DEBG 451 Reopen extent 7
Sep 22 23:22:22.973 DEBG [2] It's time to notify for 451
Sep 22 23:22:22.973 INFO Completion from [2] id:451 status:true
Sep 22 23:22:22.973 INFO [452/752] Repair commands completed
Sep 22 23:22:22.973 INFO Pop front: ReconcileIO { id: ReconciliationId(452), op: ExtentFlush { repair_id: ReconciliationId(452), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.973 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.973 INFO [0] received reconcile message
Sep 22 23:22:22.973 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(452), op: ExtentFlush { repair_id: ReconciliationId(452), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.973 INFO [0] client ExtentFlush { repair_id: ReconciliationId(452), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.973 INFO [1] received reconcile message
Sep 22 23:22:22.973 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(452), op: ExtentFlush { repair_id: ReconciliationId(452), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:22.973 INFO [1] client ExtentFlush { repair_id: ReconciliationId(452), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.973 INFO [2] received reconcile message
Sep 22 23:22:22.973 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(452), op: ExtentFlush { repair_id: ReconciliationId(452), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.973 INFO [2] client ExtentFlush { repair_id: ReconciliationId(452), extent_id: 63, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:22.973 DEBG 452 Flush extent 63 with f:2 g:2
Sep 22 23:22:22.973 DEBG Flush just extent 63 with f:2 and g:2
Sep 22 23:22:22.974 DEBG [1] It's time to notify for 452
Sep 22 23:22:22.974 INFO Completion from [1] id:452 status:true
Sep 22 23:22:22.974 INFO [453/752] Repair commands completed
Sep 22 23:22:22.974 INFO Pop front: ReconcileIO { id: ReconciliationId(453), op: ExtentClose { repair_id: ReconciliationId(453), extent_id: 63 }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.974 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.974 INFO [0] received reconcile message
Sep 22 23:22:22.974 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(453), op: ExtentClose { repair_id: ReconciliationId(453), extent_id: 63 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.974 INFO [0] client ExtentClose { repair_id: ReconciliationId(453), extent_id: 63 }
Sep 22 23:22:22.974 INFO [1] received reconcile message
Sep 22 23:22:22.974 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(453), op: ExtentClose { repair_id: ReconciliationId(453), extent_id: 63 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.974 INFO [1] client ExtentClose { repair_id: ReconciliationId(453), extent_id: 63 }
Sep 22 23:22:22.974 INFO [2] received reconcile message
Sep 22 23:22:22.974 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(453), op: ExtentClose { repair_id: ReconciliationId(453), extent_id: 63 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:22.974 INFO [2] client ExtentClose { repair_id: ReconciliationId(453), extent_id: 63 }
Sep 22 23:22:22.974 DEBG 453 Close extent 63
Sep 22 23:22:22.974 DEBG 453 Close extent 63
Sep 22 23:22:22.975 DEBG 453 Close extent 63
Sep 22 23:22:22.975 DEBG [2] It's time to notify for 453
Sep 22 23:22:22.975 INFO Completion from [2] id:453 status:true
Sep 22 23:22:22.975 INFO [454/752] Repair commands completed
Sep 22 23:22:22.975 INFO Pop front: ReconcileIO { id: ReconciliationId(454), op: ExtentRepair { repair_id: ReconciliationId(454), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:22.975 INFO Sent repair work, now wait for resp
Sep 22 23:22:22.975 INFO [0] received reconcile message
Sep 22 23:22:22.975 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(454), op: ExtentRepair { repair_id: ReconciliationId(454), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:22.975 INFO [0] client ExtentRepair { repair_id: ReconciliationId(454), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.975 INFO [0] Sending repair request ReconciliationId(454)
Sep 22 23:22:22.975 INFO [1] received reconcile message
Sep 22 23:22:22.975 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(454), op: ExtentRepair { repair_id: ReconciliationId(454), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:22.975 INFO [1] client ExtentRepair { repair_id: ReconciliationId(454), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.975 INFO [1] No action required ReconciliationId(454)
Sep 22 23:22:22.975 INFO [2] received reconcile message
Sep 22 23:22:22.975 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(454), op: ExtentRepair { repair_id: ReconciliationId(454), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:22.975 INFO [2] client ExtentRepair { repair_id: ReconciliationId(454), extent_id: 63, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:22.975 INFO [2] No action required ReconciliationId(454)
Sep 22 23:22:22.975 DEBG 454 Repair extent 63 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:22.975 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/03F.copy"
Sep 22 23:22:23.019 ERRO [2] job id 1084 saw error GenericError("test error")
Sep 22 23:22:23.019 DEBG [rc] retire 1084 clears [JobId(1083), JobId(1084)], : downstairs
Sep 22 23:22:23.034 WARN returning error on flush!
Sep 22 23:22:23.034 DEBG Flush     :1086 extent_limit None deps:[JobId(1085), JobId(1084)] res:false f:32 g:1
Sep 22 23:22:23.034 INFO [lossy] skipping 1087
Sep 22 23:22:23.034 DEBG Flush     :1086 extent_limit None deps:[JobId(1085), JobId(1084)] res:true f:32 g:1
Sep 22 23:22:23.040 DEBG Read      :1087 deps:[JobId(1086)] res:true
Sep 22 23:22:23.042 INFO accepted connection, remote_addr: 127.0.0.1:40604, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.042 TRCE incoming request, uri: /extent/63/files, method: GET, req_id: 371d99ed-57ad-4b29-a15e-3fc077feb00d, remote_addr: 127.0.0.1:40604, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.042 INFO request completed, latency_us: 267, response_code: 200, uri: /extent/63/files, method: GET, req_id: 371d99ed-57ad-4b29-a15e-3fc077feb00d, remote_addr: 127.0.0.1:40604, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.043 INFO eid:63 Found repair files: ["03F", "03F.db"]
Sep 22 23:22:23.043 TRCE incoming request, uri: /newextent/63/data, method: GET, req_id: 2efc4e3f-0f46-4d13-bd24-cbae4417ec1b, remote_addr: 127.0.0.1:40604, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.043 INFO request completed, latency_us: 370, response_code: 200, uri: /newextent/63/data, method: GET, req_id: 2efc4e3f-0f46-4d13-bd24-cbae4417ec1b, remote_addr: 127.0.0.1:40604, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.048 TRCE incoming request, uri: /newextent/63/db, method: GET, req_id: 28e2ab0b-3c3f-4120-b526-816e5c996973, remote_addr: 127.0.0.1:40604, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.048 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/63/db, method: GET, req_id: 28e2ab0b-3c3f-4120-b526-816e5c996973, remote_addr: 127.0.0.1:40604, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.050 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/03F.copy" to "/tmp/downstairs-zrMnlo6G/00/000/03F.replace"
Sep 22 23:22:23.050 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.051 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/03F.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.051 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03F"
Sep 22 23:22:23.051 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03F.db"
Sep 22 23:22:23.051 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.051 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/03F.replace" to "/tmp/downstairs-zrMnlo6G/00/000/03F.completed"
Sep 22 23:22:23.051 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.051 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.051 DEBG [0] It's time to notify for 454
Sep 22 23:22:23.052 INFO Completion from [0] id:454 status:true
Sep 22 23:22:23.052 INFO [455/752] Repair commands completed
Sep 22 23:22:23.052 INFO Pop front: ReconcileIO { id: ReconciliationId(455), op: ExtentReopen { repair_id: ReconciliationId(455), extent_id: 63 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.052 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.052 INFO [0] received reconcile message
Sep 22 23:22:23.052 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(455), op: ExtentReopen { repair_id: ReconciliationId(455), extent_id: 63 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.052 INFO [0] client ExtentReopen { repair_id: ReconciliationId(455), extent_id: 63 }
Sep 22 23:22:23.052 INFO [1] received reconcile message
Sep 22 23:22:23.052 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(455), op: ExtentReopen { repair_id: ReconciliationId(455), extent_id: 63 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.052 INFO [1] client ExtentReopen { repair_id: ReconciliationId(455), extent_id: 63 }
Sep 22 23:22:23.052 INFO [2] received reconcile message
Sep 22 23:22:23.052 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(455), op: ExtentReopen { repair_id: ReconciliationId(455), extent_id: 63 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.052 INFO [2] client ExtentReopen { repair_id: ReconciliationId(455), extent_id: 63 }
Sep 22 23:22:23.052 DEBG 455 Reopen extent 63
Sep 22 23:22:23.053 DEBG 455 Reopen extent 63
Sep 22 23:22:23.053 DEBG 455 Reopen extent 63
Sep 22 23:22:23.054 DEBG [2] It's time to notify for 455
Sep 22 23:22:23.054 INFO Completion from [2] id:455 status:true
Sep 22 23:22:23.054 INFO [456/752] Repair commands completed
Sep 22 23:22:23.054 INFO Pop front: ReconcileIO { id: ReconciliationId(456), op: ExtentFlush { repair_id: ReconciliationId(456), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.054 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.054 INFO [0] received reconcile message
Sep 22 23:22:23.054 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(456), op: ExtentFlush { repair_id: ReconciliationId(456), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.054 INFO [0] client ExtentFlush { repair_id: ReconciliationId(456), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.054 INFO [1] received reconcile message
Sep 22 23:22:23.054 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(456), op: ExtentFlush { repair_id: ReconciliationId(456), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.054 INFO [1] client ExtentFlush { repair_id: ReconciliationId(456), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.054 INFO [2] received reconcile message
Sep 22 23:22:23.054 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(456), op: ExtentFlush { repair_id: ReconciliationId(456), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.054 INFO [2] client ExtentFlush { repair_id: ReconciliationId(456), extent_id: 58, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.054 DEBG 456 Flush extent 58 with f:2 g:2
Sep 22 23:22:23.054 DEBG Flush just extent 58 with f:2 and g:2
Sep 22 23:22:23.055 DEBG [1] It's time to notify for 456
Sep 22 23:22:23.055 INFO Completion from [1] id:456 status:true
Sep 22 23:22:23.055 INFO [457/752] Repair commands completed
Sep 22 23:22:23.055 INFO Pop front: ReconcileIO { id: ReconciliationId(457), op: ExtentClose { repair_id: ReconciliationId(457), extent_id: 58 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.055 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.055 INFO [0] received reconcile message
Sep 22 23:22:23.055 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(457), op: ExtentClose { repair_id: ReconciliationId(457), extent_id: 58 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.055 INFO [0] client ExtentClose { repair_id: ReconciliationId(457), extent_id: 58 }
Sep 22 23:22:23.055 INFO [1] received reconcile message
Sep 22 23:22:23.055 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(457), op: ExtentClose { repair_id: ReconciliationId(457), extent_id: 58 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.055 INFO [1] client ExtentClose { repair_id: ReconciliationId(457), extent_id: 58 }
Sep 22 23:22:23.055 INFO [2] received reconcile message
Sep 22 23:22:23.055 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(457), op: ExtentClose { repair_id: ReconciliationId(457), extent_id: 58 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.055 INFO [2] client ExtentClose { repair_id: ReconciliationId(457), extent_id: 58 }
Sep 22 23:22:23.055 DEBG 457 Close extent 58
Sep 22 23:22:23.055 DEBG 457 Close extent 58
Sep 22 23:22:23.056 DEBG 457 Close extent 58
Sep 22 23:22:23.056 DEBG [2] It's time to notify for 457
Sep 22 23:22:23.056 INFO Completion from [2] id:457 status:true
Sep 22 23:22:23.056 INFO [458/752] Repair commands completed
Sep 22 23:22:23.056 INFO Pop front: ReconcileIO { id: ReconciliationId(458), op: ExtentRepair { repair_id: ReconciliationId(458), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.056 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.056 INFO [0] received reconcile message
Sep 22 23:22:23.056 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(458), op: ExtentRepair { repair_id: ReconciliationId(458), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.056 INFO [0] client ExtentRepair { repair_id: ReconciliationId(458), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.056 INFO [0] Sending repair request ReconciliationId(458)
Sep 22 23:22:23.056 INFO [1] received reconcile message
Sep 22 23:22:23.056 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(458), op: ExtentRepair { repair_id: ReconciliationId(458), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.056 INFO [1] client ExtentRepair { repair_id: ReconciliationId(458), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.056 INFO [1] No action required ReconciliationId(458)
Sep 22 23:22:23.056 INFO [2] received reconcile message
Sep 22 23:22:23.056 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(458), op: ExtentRepair { repair_id: ReconciliationId(458), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.056 INFO [2] client ExtentRepair { repair_id: ReconciliationId(458), extent_id: 58, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.056 INFO [2] No action required ReconciliationId(458)
Sep 22 23:22:23.057 DEBG 458 Repair extent 58 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.057 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/03A.copy"
Sep 22 23:22:23.118 INFO accepted connection, remote_addr: 127.0.0.1:33238, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.119 TRCE incoming request, uri: /extent/58/files, method: GET, req_id: fdff48ca-2f8d-4758-bd73-cf22271aa9a5, remote_addr: 127.0.0.1:33238, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.119 INFO request completed, latency_us: 208, response_code: 200, uri: /extent/58/files, method: GET, req_id: fdff48ca-2f8d-4758-bd73-cf22271aa9a5, remote_addr: 127.0.0.1:33238, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.119 INFO eid:58 Found repair files: ["03A", "03A.db"]
Sep 22 23:22:23.119 TRCE incoming request, uri: /newextent/58/data, method: GET, req_id: 577b5e80-c943-4541-a2ad-66739862b242, remote_addr: 127.0.0.1:33238, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.120 INFO request completed, latency_us: 308, response_code: 200, uri: /newextent/58/data, method: GET, req_id: 577b5e80-c943-4541-a2ad-66739862b242, remote_addr: 127.0.0.1:33238, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.124 TRCE incoming request, uri: /newextent/58/db, method: GET, req_id: 89bc34f9-1fb7-468b-96f9-4ec832ee626e, remote_addr: 127.0.0.1:33238, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.125 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/58/db, method: GET, req_id: 89bc34f9-1fb7-468b-96f9-4ec832ee626e, remote_addr: 127.0.0.1:33238, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.126 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/03A.copy" to "/tmp/downstairs-zrMnlo6G/00/000/03A.replace"
Sep 22 23:22:23.126 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.127 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/03A.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.127 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03A"
Sep 22 23:22:23.127 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/03A.db"
Sep 22 23:22:23.127 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.127 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/03A.replace" to "/tmp/downstairs-zrMnlo6G/00/000/03A.completed"
Sep 22 23:22:23.127 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.127 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.127 DEBG [0] It's time to notify for 458
Sep 22 23:22:23.128 INFO Completion from [0] id:458 status:true
Sep 22 23:22:23.128 INFO [459/752] Repair commands completed
Sep 22 23:22:23.128 INFO Pop front: ReconcileIO { id: ReconciliationId(459), op: ExtentReopen { repair_id: ReconciliationId(459), extent_id: 58 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.128 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.128 INFO [0] received reconcile message
Sep 22 23:22:23.128 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(459), op: ExtentReopen { repair_id: ReconciliationId(459), extent_id: 58 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.128 INFO [0] client ExtentReopen { repair_id: ReconciliationId(459), extent_id: 58 }
Sep 22 23:22:23.128 INFO [1] received reconcile message
Sep 22 23:22:23.128 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(459), op: ExtentReopen { repair_id: ReconciliationId(459), extent_id: 58 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.128 INFO [1] client ExtentReopen { repair_id: ReconciliationId(459), extent_id: 58 }
Sep 22 23:22:23.128 INFO [2] received reconcile message
Sep 22 23:22:23.128 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(459), op: ExtentReopen { repair_id: ReconciliationId(459), extent_id: 58 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.128 INFO [2] client ExtentReopen { repair_id: ReconciliationId(459), extent_id: 58 }
Sep 22 23:22:23.128 DEBG 459 Reopen extent 58
Sep 22 23:22:23.129 DEBG 459 Reopen extent 58
Sep 22 23:22:23.129 DEBG 459 Reopen extent 58
Sep 22 23:22:23.130 DEBG [2] It's time to notify for 459
Sep 22 23:22:23.130 INFO Completion from [2] id:459 status:true
Sep 22 23:22:23.130 INFO [460/752] Repair commands completed
Sep 22 23:22:23.130 INFO Pop front: ReconcileIO { id: ReconciliationId(460), op: ExtentFlush { repair_id: ReconciliationId(460), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.130 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.130 INFO [0] received reconcile message
Sep 22 23:22:23.130 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(460), op: ExtentFlush { repair_id: ReconciliationId(460), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.130 INFO [0] client ExtentFlush { repair_id: ReconciliationId(460), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.130 INFO [1] received reconcile message
Sep 22 23:22:23.130 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(460), op: ExtentFlush { repair_id: ReconciliationId(460), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.130 INFO [1] client ExtentFlush { repair_id: ReconciliationId(460), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.130 INFO [2] received reconcile message
Sep 22 23:22:23.130 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(460), op: ExtentFlush { repair_id: ReconciliationId(460), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.130 INFO [2] client ExtentFlush { repair_id: ReconciliationId(460), extent_id: 96, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.130 DEBG 460 Flush extent 96 with f:2 g:2
Sep 22 23:22:23.130 DEBG Flush just extent 96 with f:2 and g:2
Sep 22 23:22:23.130 DEBG [1] It's time to notify for 460
Sep 22 23:22:23.130 INFO Completion from [1] id:460 status:true
Sep 22 23:22:23.130 INFO [461/752] Repair commands completed
Sep 22 23:22:23.130 INFO Pop front: ReconcileIO { id: ReconciliationId(461), op: ExtentClose { repair_id: ReconciliationId(461), extent_id: 96 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.130 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.130 INFO [0] received reconcile message
Sep 22 23:22:23.130 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(461), op: ExtentClose { repair_id: ReconciliationId(461), extent_id: 96 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.130 INFO [0] client ExtentClose { repair_id: ReconciliationId(461), extent_id: 96 }
Sep 22 23:22:23.131 INFO [1] received reconcile message
Sep 22 23:22:23.131 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(461), op: ExtentClose { repair_id: ReconciliationId(461), extent_id: 96 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.131 INFO [1] client ExtentClose { repair_id: ReconciliationId(461), extent_id: 96 }
Sep 22 23:22:23.131 INFO [2] received reconcile message
Sep 22 23:22:23.131 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(461), op: ExtentClose { repair_id: ReconciliationId(461), extent_id: 96 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.131 INFO [2] client ExtentClose { repair_id: ReconciliationId(461), extent_id: 96 }
Sep 22 23:22:23.131 DEBG 461 Close extent 96
Sep 22 23:22:23.131 DEBG 461 Close extent 96
Sep 22 23:22:23.131 DEBG 461 Close extent 96
Sep 22 23:22:23.132 DEBG [2] It's time to notify for 461
Sep 22 23:22:23.132 INFO Completion from [2] id:461 status:true
Sep 22 23:22:23.132 INFO [462/752] Repair commands completed
Sep 22 23:22:23.132 INFO Pop front: ReconcileIO { id: ReconciliationId(462), op: ExtentRepair { repair_id: ReconciliationId(462), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.132 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.132 INFO [0] received reconcile message
Sep 22 23:22:23.132 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(462), op: ExtentRepair { repair_id: ReconciliationId(462), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.132 INFO [0] client ExtentRepair { repair_id: ReconciliationId(462), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.132 INFO [0] Sending repair request ReconciliationId(462)
Sep 22 23:22:23.132 INFO [1] received reconcile message
Sep 22 23:22:23.132 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(462), op: ExtentRepair { repair_id: ReconciliationId(462), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.132 INFO [1] client ExtentRepair { repair_id: ReconciliationId(462), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.132 INFO [1] No action required ReconciliationId(462)
Sep 22 23:22:23.132 INFO [2] received reconcile message
Sep 22 23:22:23.132 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(462), op: ExtentRepair { repair_id: ReconciliationId(462), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.132 INFO [2] client ExtentRepair { repair_id: ReconciliationId(462), extent_id: 96, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.132 INFO [2] No action required ReconciliationId(462)
Sep 22 23:22:23.132 DEBG 462 Repair extent 96 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.132 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/060.copy"
Sep 22 23:22:23.198 INFO accepted connection, remote_addr: 127.0.0.1:54809, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.198 TRCE incoming request, uri: /extent/96/files, method: GET, req_id: a4c6c8dc-b70e-4eb6-b050-102bf7b0a29a, remote_addr: 127.0.0.1:54809, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.198 INFO request completed, latency_us: 206, response_code: 200, uri: /extent/96/files, method: GET, req_id: a4c6c8dc-b70e-4eb6-b050-102bf7b0a29a, remote_addr: 127.0.0.1:54809, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.198 INFO eid:96 Found repair files: ["060", "060.db"]
Sep 22 23:22:23.199 TRCE incoming request, uri: /newextent/96/data, method: GET, req_id: 148cf95a-1c4d-4d50-9aeb-d9a7f0df55cc, remote_addr: 127.0.0.1:54809, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.199 INFO request completed, latency_us: 261, response_code: 200, uri: /newextent/96/data, method: GET, req_id: 148cf95a-1c4d-4d50-9aeb-d9a7f0df55cc, remote_addr: 127.0.0.1:54809, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.204 TRCE incoming request, uri: /newextent/96/db, method: GET, req_id: 1655e12a-5268-448a-aeae-739023cd5943, remote_addr: 127.0.0.1:54809, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.205 INFO request completed, latency_us: 319, response_code: 200, uri: /newextent/96/db, method: GET, req_id: 1655e12a-5268-448a-aeae-739023cd5943, remote_addr: 127.0.0.1:54809, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.206 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/060.copy" to "/tmp/downstairs-zrMnlo6G/00/000/060.replace"
Sep 22 23:22:23.206 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.207 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/060.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.207 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/060"
Sep 22 23:22:23.207 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/060.db"
Sep 22 23:22:23.208 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.208 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/060.replace" to "/tmp/downstairs-zrMnlo6G/00/000/060.completed"
Sep 22 23:22:23.208 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.208 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.208 DEBG [0] It's time to notify for 462
Sep 22 23:22:23.208 INFO Completion from [0] id:462 status:true
Sep 22 23:22:23.208 INFO [463/752] Repair commands completed
Sep 22 23:22:23.208 INFO Pop front: ReconcileIO { id: ReconciliationId(463), op: ExtentReopen { repair_id: ReconciliationId(463), extent_id: 96 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.208 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.208 INFO [0] received reconcile message
Sep 22 23:22:23.208 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(463), op: ExtentReopen { repair_id: ReconciliationId(463), extent_id: 96 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.208 INFO [0] client ExtentReopen { repair_id: ReconciliationId(463), extent_id: 96 }
Sep 22 23:22:23.208 INFO [1] received reconcile message
Sep 22 23:22:23.208 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(463), op: ExtentReopen { repair_id: ReconciliationId(463), extent_id: 96 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.208 INFO [1] client ExtentReopen { repair_id: ReconciliationId(463), extent_id: 96 }
Sep 22 23:22:23.208 INFO [2] received reconcile message
Sep 22 23:22:23.208 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(463), op: ExtentReopen { repair_id: ReconciliationId(463), extent_id: 96 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.208 INFO [2] client ExtentReopen { repair_id: ReconciliationId(463), extent_id: 96 }
Sep 22 23:22:23.209 DEBG 463 Reopen extent 96
Sep 22 23:22:23.209 DEBG 463 Reopen extent 96
Sep 22 23:22:23.210 DEBG 463 Reopen extent 96
Sep 22 23:22:23.210 DEBG [2] It's time to notify for 463
Sep 22 23:22:23.211 INFO Completion from [2] id:463 status:true
Sep 22 23:22:23.211 INFO [464/752] Repair commands completed
Sep 22 23:22:23.211 INFO Pop front: ReconcileIO { id: ReconciliationId(464), op: ExtentFlush { repair_id: ReconciliationId(464), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.211 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.211 INFO [0] received reconcile message
Sep 22 23:22:23.211 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(464), op: ExtentFlush { repair_id: ReconciliationId(464), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.211 INFO [0] client ExtentFlush { repair_id: ReconciliationId(464), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.211 INFO [1] received reconcile message
Sep 22 23:22:23.211 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(464), op: ExtentFlush { repair_id: ReconciliationId(464), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.211 INFO [1] client ExtentFlush { repair_id: ReconciliationId(464), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.211 INFO [2] received reconcile message
Sep 22 23:22:23.211 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(464), op: ExtentFlush { repair_id: ReconciliationId(464), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.211 INFO [2] client ExtentFlush { repair_id: ReconciliationId(464), extent_id: 40, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.211 DEBG 464 Flush extent 40 with f:2 g:2
Sep 22 23:22:23.211 DEBG Flush just extent 40 with f:2 and g:2
Sep 22 23:22:23.211 DEBG [1] It's time to notify for 464
Sep 22 23:22:23.211 INFO Completion from [1] id:464 status:true
Sep 22 23:22:23.211 INFO [465/752] Repair commands completed
Sep 22 23:22:23.211 INFO Pop front: ReconcileIO { id: ReconciliationId(465), op: ExtentClose { repair_id: ReconciliationId(465), extent_id: 40 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.211 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.211 INFO [0] received reconcile message
Sep 22 23:22:23.211 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(465), op: ExtentClose { repair_id: ReconciliationId(465), extent_id: 40 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.211 INFO [0] client ExtentClose { repair_id: ReconciliationId(465), extent_id: 40 }
Sep 22 23:22:23.211 INFO [1] received reconcile message
Sep 22 23:22:23.211 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(465), op: ExtentClose { repair_id: ReconciliationId(465), extent_id: 40 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.211 INFO [1] client ExtentClose { repair_id: ReconciliationId(465), extent_id: 40 }
Sep 22 23:22:23.211 INFO [2] received reconcile message
Sep 22 23:22:23.211 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(465), op: ExtentClose { repair_id: ReconciliationId(465), extent_id: 40 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.211 INFO [2] client ExtentClose { repair_id: ReconciliationId(465), extent_id: 40 }
Sep 22 23:22:23.212 DEBG 465 Close extent 40
Sep 22 23:22:23.212 DEBG 465 Close extent 40
Sep 22 23:22:23.212 DEBG 465 Close extent 40
Sep 22 23:22:23.212 DEBG [2] It's time to notify for 465
Sep 22 23:22:23.213 INFO Completion from [2] id:465 status:true
Sep 22 23:22:23.213 INFO [466/752] Repair commands completed
Sep 22 23:22:23.213 INFO Pop front: ReconcileIO { id: ReconciliationId(466), op: ExtentRepair { repair_id: ReconciliationId(466), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.213 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.213 INFO [0] received reconcile message
Sep 22 23:22:23.213 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(466), op: ExtentRepair { repair_id: ReconciliationId(466), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.213 INFO [0] client ExtentRepair { repair_id: ReconciliationId(466), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.213 INFO [0] Sending repair request ReconciliationId(466)
Sep 22 23:22:23.213 INFO [1] received reconcile message
Sep 22 23:22:23.213 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(466), op: ExtentRepair { repair_id: ReconciliationId(466), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.213 INFO [1] client ExtentRepair { repair_id: ReconciliationId(466), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.213 INFO [1] No action required ReconciliationId(466)
Sep 22 23:22:23.213 INFO [2] received reconcile message
Sep 22 23:22:23.213 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(466), op: ExtentRepair { repair_id: ReconciliationId(466), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.213 INFO [2] client ExtentRepair { repair_id: ReconciliationId(466), extent_id: 40, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.213 INFO [2] No action required ReconciliationId(466)
Sep 22 23:22:23.213 DEBG 466 Repair extent 40 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.213 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/028.copy"
Sep 22 23:22:23.277 INFO accepted connection, remote_addr: 127.0.0.1:49761, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.277 TRCE incoming request, uri: /extent/40/files, method: GET, req_id: 358fc8b2-0693-49b6-9aff-b0a4aa32da26, remote_addr: 127.0.0.1:49761, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.277 INFO request completed, latency_us: 278, response_code: 200, uri: /extent/40/files, method: GET, req_id: 358fc8b2-0693-49b6-9aff-b0a4aa32da26, remote_addr: 127.0.0.1:49761, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.278 INFO eid:40 Found repair files: ["028", "028.db"]
Sep 22 23:22:23.278 TRCE incoming request, uri: /newextent/40/data, method: GET, req_id: 0cc30ce9-9b8e-406b-b294-8db10eb7a615, remote_addr: 127.0.0.1:49761, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.278 INFO request completed, latency_us: 371, response_code: 200, uri: /newextent/40/data, method: GET, req_id: 0cc30ce9-9b8e-406b-b294-8db10eb7a615, remote_addr: 127.0.0.1:49761, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.283 TRCE incoming request, uri: /newextent/40/db, method: GET, req_id: c043eba6-ae14-46b1-a1e8-c0427fd8903f, remote_addr: 127.0.0.1:49761, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.283 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/40/db, method: GET, req_id: c043eba6-ae14-46b1-a1e8-c0427fd8903f, remote_addr: 127.0.0.1:49761, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.284 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/028.copy" to "/tmp/downstairs-zrMnlo6G/00/000/028.replace"
Sep 22 23:22:23.284 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.286 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/028.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.286 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/028"
Sep 22 23:22:23.286 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/028.db"
Sep 22 23:22:23.286 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.286 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/028.replace" to "/tmp/downstairs-zrMnlo6G/00/000/028.completed"
Sep 22 23:22:23.286 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.286 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.286 DEBG [0] It's time to notify for 466
Sep 22 23:22:23.286 INFO Completion from [0] id:466 status:true
Sep 22 23:22:23.286 INFO [467/752] Repair commands completed
Sep 22 23:22:23.286 INFO Pop front: ReconcileIO { id: ReconciliationId(467), op: ExtentReopen { repair_id: ReconciliationId(467), extent_id: 40 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.287 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.287 INFO [0] received reconcile message
Sep 22 23:22:23.287 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(467), op: ExtentReopen { repair_id: ReconciliationId(467), extent_id: 40 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.287 INFO [0] client ExtentReopen { repair_id: ReconciliationId(467), extent_id: 40 }
Sep 22 23:22:23.287 INFO [1] received reconcile message
Sep 22 23:22:23.287 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(467), op: ExtentReopen { repair_id: ReconciliationId(467), extent_id: 40 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.287 INFO [1] client ExtentReopen { repair_id: ReconciliationId(467), extent_id: 40 }
Sep 22 23:22:23.287 INFO [2] received reconcile message
Sep 22 23:22:23.287 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(467), op: ExtentReopen { repair_id: ReconciliationId(467), extent_id: 40 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.287 INFO [2] client ExtentReopen { repair_id: ReconciliationId(467), extent_id: 40 }
Sep 22 23:22:23.287 DEBG 467 Reopen extent 40
Sep 22 23:22:23.288 DEBG 467 Reopen extent 40
Sep 22 23:22:23.288 DEBG 467 Reopen extent 40
Sep 22 23:22:23.289 DEBG [2] It's time to notify for 467
Sep 22 23:22:23.289 INFO Completion from [2] id:467 status:true
Sep 22 23:22:23.289 INFO [468/752] Repair commands completed
Sep 22 23:22:23.289 INFO Pop front: ReconcileIO { id: ReconciliationId(468), op: ExtentFlush { repair_id: ReconciliationId(468), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.289 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.289 INFO [0] received reconcile message
Sep 22 23:22:23.289 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(468), op: ExtentFlush { repair_id: ReconciliationId(468), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.289 INFO [0] client ExtentFlush { repair_id: ReconciliationId(468), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.289 INFO [1] received reconcile message
Sep 22 23:22:23.289 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(468), op: ExtentFlush { repair_id: ReconciliationId(468), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.289 INFO [1] client ExtentFlush { repair_id: ReconciliationId(468), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.289 INFO [2] received reconcile message
Sep 22 23:22:23.289 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(468), op: ExtentFlush { repair_id: ReconciliationId(468), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.289 INFO [2] client ExtentFlush { repair_id: ReconciliationId(468), extent_id: 80, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.289 DEBG 468 Flush extent 80 with f:2 g:2
Sep 22 23:22:23.289 DEBG Flush just extent 80 with f:2 and g:2
Sep 22 23:22:23.289 DEBG [1] It's time to notify for 468
Sep 22 23:22:23.290 INFO Completion from [1] id:468 status:true
Sep 22 23:22:23.290 INFO [469/752] Repair commands completed
Sep 22 23:22:23.290 INFO Pop front: ReconcileIO { id: ReconciliationId(469), op: ExtentClose { repair_id: ReconciliationId(469), extent_id: 80 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.290 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.290 INFO [0] received reconcile message
Sep 22 23:22:23.290 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(469), op: ExtentClose { repair_id: ReconciliationId(469), extent_id: 80 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.290 INFO [0] client ExtentClose { repair_id: ReconciliationId(469), extent_id: 80 }
Sep 22 23:22:23.290 INFO [1] received reconcile message
Sep 22 23:22:23.290 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(469), op: ExtentClose { repair_id: ReconciliationId(469), extent_id: 80 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.290 INFO [1] client ExtentClose { repair_id: ReconciliationId(469), extent_id: 80 }
Sep 22 23:22:23.290 INFO [2] received reconcile message
Sep 22 23:22:23.290 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(469), op: ExtentClose { repair_id: ReconciliationId(469), extent_id: 80 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.290 INFO [2] client ExtentClose { repair_id: ReconciliationId(469), extent_id: 80 }
Sep 22 23:22:23.290 DEBG 469 Close extent 80
Sep 22 23:22:23.290 DEBG 469 Close extent 80
Sep 22 23:22:23.291 DEBG 469 Close extent 80
Sep 22 23:22:23.291 DEBG [2] It's time to notify for 469
Sep 22 23:22:23.291 INFO Completion from [2] id:469 status:true
Sep 22 23:22:23.291 INFO [470/752] Repair commands completed
Sep 22 23:22:23.291 INFO Pop front: ReconcileIO { id: ReconciliationId(470), op: ExtentRepair { repair_id: ReconciliationId(470), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.291 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.291 INFO [0] received reconcile message
Sep 22 23:22:23.291 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(470), op: ExtentRepair { repair_id: ReconciliationId(470), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.291 INFO [0] client ExtentRepair { repair_id: ReconciliationId(470), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.291 INFO [0] Sending repair request ReconciliationId(470)
Sep 22 23:22:23.291 INFO [1] received reconcile message
Sep 22 23:22:23.291 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(470), op: ExtentRepair { repair_id: ReconciliationId(470), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.291 INFO [1] client ExtentRepair { repair_id: ReconciliationId(470), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.291 INFO [1] No action required ReconciliationId(470)
Sep 22 23:22:23.291 INFO [2] received reconcile message
Sep 22 23:22:23.291 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(470), op: ExtentRepair { repair_id: ReconciliationId(470), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.291 INFO [2] client ExtentRepair { repair_id: ReconciliationId(470), extent_id: 80, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.291 INFO [2] No action required ReconciliationId(470)
Sep 22 23:22:23.291 DEBG 470 Repair extent 80 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.292 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/050.copy"
Sep 22 23:22:23.357 INFO accepted connection, remote_addr: 127.0.0.1:49808, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.357 TRCE incoming request, uri: /extent/80/files, method: GET, req_id: ebfc2499-6f2b-4b77-a197-c6d778e3038a, remote_addr: 127.0.0.1:49808, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.358 INFO request completed, latency_us: 227, response_code: 200, uri: /extent/80/files, method: GET, req_id: ebfc2499-6f2b-4b77-a197-c6d778e3038a, remote_addr: 127.0.0.1:49808, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.358 INFO eid:80 Found repair files: ["050", "050.db"]
Sep 22 23:22:23.358 TRCE incoming request, uri: /newextent/80/data, method: GET, req_id: 637edeab-0f07-4dc4-82f1-cae16a5a5a4d, remote_addr: 127.0.0.1:49808, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.359 INFO request completed, latency_us: 319, response_code: 200, uri: /newextent/80/data, method: GET, req_id: 637edeab-0f07-4dc4-82f1-cae16a5a5a4d, remote_addr: 127.0.0.1:49808, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.363 TRCE incoming request, uri: /newextent/80/db, method: GET, req_id: 31e856b4-e01b-4b04-9c15-6bc8c746bc20, remote_addr: 127.0.0.1:49808, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.364 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/80/db, method: GET, req_id: 31e856b4-e01b-4b04-9c15-6bc8c746bc20, remote_addr: 127.0.0.1:49808, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.365 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/050.copy" to "/tmp/downstairs-zrMnlo6G/00/000/050.replace"
Sep 22 23:22:23.365 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.366 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/050.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.366 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/050"
Sep 22 23:22:23.366 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/050.db"
Sep 22 23:22:23.366 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.366 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/050.replace" to "/tmp/downstairs-zrMnlo6G/00/000/050.completed"
Sep 22 23:22:23.366 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.366 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.366 DEBG [0] It's time to notify for 470
Sep 22 23:22:23.366 INFO Completion from [0] id:470 status:true
Sep 22 23:22:23.366 INFO [471/752] Repair commands completed
Sep 22 23:22:23.367 INFO Pop front: ReconcileIO { id: ReconciliationId(471), op: ExtentReopen { repair_id: ReconciliationId(471), extent_id: 80 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.367 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.367 INFO [0] received reconcile message
Sep 22 23:22:23.367 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(471), op: ExtentReopen { repair_id: ReconciliationId(471), extent_id: 80 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.367 INFO [0] client ExtentReopen { repair_id: ReconciliationId(471), extent_id: 80 }
Sep 22 23:22:23.367 INFO [1] received reconcile message
Sep 22 23:22:23.367 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(471), op: ExtentReopen { repair_id: ReconciliationId(471), extent_id: 80 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.367 INFO [1] client ExtentReopen { repair_id: ReconciliationId(471), extent_id: 80 }
Sep 22 23:22:23.367 INFO [2] received reconcile message
Sep 22 23:22:23.367 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(471), op: ExtentReopen { repair_id: ReconciliationId(471), extent_id: 80 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.367 INFO [2] client ExtentReopen { repair_id: ReconciliationId(471), extent_id: 80 }
Sep 22 23:22:23.367 DEBG 471 Reopen extent 80
Sep 22 23:22:23.368 DEBG 471 Reopen extent 80
Sep 22 23:22:23.368 DEBG 471 Reopen extent 80
Sep 22 23:22:23.369 DEBG [2] It's time to notify for 471
Sep 22 23:22:23.369 INFO Completion from [2] id:471 status:true
Sep 22 23:22:23.369 INFO [472/752] Repair commands completed
Sep 22 23:22:23.369 INFO Pop front: ReconcileIO { id: ReconciliationId(472), op: ExtentFlush { repair_id: ReconciliationId(472), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.369 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.369 INFO [0] received reconcile message
Sep 22 23:22:23.369 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(472), op: ExtentFlush { repair_id: ReconciliationId(472), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.369 INFO [0] client ExtentFlush { repair_id: ReconciliationId(472), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.369 INFO [1] received reconcile message
Sep 22 23:22:23.369 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(472), op: ExtentFlush { repair_id: ReconciliationId(472), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.369 INFO [1] client ExtentFlush { repair_id: ReconciliationId(472), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.369 INFO [2] received reconcile message
Sep 22 23:22:23.369 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(472), op: ExtentFlush { repair_id: ReconciliationId(472), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.369 INFO [2] client ExtentFlush { repair_id: ReconciliationId(472), extent_id: 149, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.369 DEBG 472 Flush extent 149 with f:2 g:2
Sep 22 23:22:23.369 DEBG Flush just extent 149 with f:2 and g:2
Sep 22 23:22:23.369 DEBG [1] It's time to notify for 472
Sep 22 23:22:23.369 INFO Completion from [1] id:472 status:true
Sep 22 23:22:23.369 INFO [473/752] Repair commands completed
Sep 22 23:22:23.369 INFO Pop front: ReconcileIO { id: ReconciliationId(473), op: ExtentClose { repair_id: ReconciliationId(473), extent_id: 149 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.369 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.369 INFO [0] received reconcile message
Sep 22 23:22:23.370 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(473), op: ExtentClose { repair_id: ReconciliationId(473), extent_id: 149 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.370 INFO [0] client ExtentClose { repair_id: ReconciliationId(473), extent_id: 149 }
Sep 22 23:22:23.370 INFO [1] received reconcile message
Sep 22 23:22:23.370 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(473), op: ExtentClose { repair_id: ReconciliationId(473), extent_id: 149 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.370 INFO [1] client ExtentClose { repair_id: ReconciliationId(473), extent_id: 149 }
Sep 22 23:22:23.370 INFO [2] received reconcile message
Sep 22 23:22:23.370 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(473), op: ExtentClose { repair_id: ReconciliationId(473), extent_id: 149 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.370 INFO [2] client ExtentClose { repair_id: ReconciliationId(473), extent_id: 149 }
Sep 22 23:22:23.370 DEBG 473 Close extent 149
Sep 22 23:22:23.370 DEBG 473 Close extent 149
Sep 22 23:22:23.370 DEBG 473 Close extent 149
Sep 22 23:22:23.371 DEBG [2] It's time to notify for 473
Sep 22 23:22:23.371 INFO Completion from [2] id:473 status:true
Sep 22 23:22:23.371 INFO [474/752] Repair commands completed
Sep 22 23:22:23.371 INFO Pop front: ReconcileIO { id: ReconciliationId(474), op: ExtentRepair { repair_id: ReconciliationId(474), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.371 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.371 INFO [0] received reconcile message
Sep 22 23:22:23.371 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(474), op: ExtentRepair { repair_id: ReconciliationId(474), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.371 INFO [0] client ExtentRepair { repair_id: ReconciliationId(474), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.371 INFO [0] Sending repair request ReconciliationId(474)
Sep 22 23:22:23.371 INFO [1] received reconcile message
Sep 22 23:22:23.371 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(474), op: ExtentRepair { repair_id: ReconciliationId(474), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.371 INFO [1] client ExtentRepair { repair_id: ReconciliationId(474), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.371 INFO [1] No action required ReconciliationId(474)
Sep 22 23:22:23.371 INFO [2] received reconcile message
Sep 22 23:22:23.371 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(474), op: ExtentRepair { repair_id: ReconciliationId(474), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.371 INFO [2] client ExtentRepair { repair_id: ReconciliationId(474), extent_id: 149, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.371 INFO [2] No action required ReconciliationId(474)
Sep 22 23:22:23.371 DEBG 474 Repair extent 149 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.371 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/095.copy"
Sep 22 23:22:23.436 INFO accepted connection, remote_addr: 127.0.0.1:59607, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.437 TRCE incoming request, uri: /extent/149/files, method: GET, req_id: da31bb3f-0a5d-4149-96fa-a915d6851d35, remote_addr: 127.0.0.1:59607, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.437 INFO request completed, latency_us: 200, response_code: 200, uri: /extent/149/files, method: GET, req_id: da31bb3f-0a5d-4149-96fa-a915d6851d35, remote_addr: 127.0.0.1:59607, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.437 INFO eid:149 Found repair files: ["095", "095.db"]
Sep 22 23:22:23.437 TRCE incoming request, uri: /newextent/149/data, method: GET, req_id: 1e084aaa-5e86-47f7-9fb6-cd9c3a5756aa, remote_addr: 127.0.0.1:59607, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.438 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/149/data, method: GET, req_id: 1e084aaa-5e86-47f7-9fb6-cd9c3a5756aa, remote_addr: 127.0.0.1:59607, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.442 ERRO [1] job id 1086 saw error GenericError("test error")
Sep 22 23:22:23.442 DEBG IO Flush 1088 has deps [JobId(1087), JobId(1086)]
Sep 22 23:22:23.442 DEBG Flush     :1086 extent_limit None deps:[JobId(1085), JobId(1084)] res:true f:32 g:1
Sep 22 23:22:23.442 TRCE incoming request, uri: /newextent/149/db, method: GET, req_id: 66eb705b-2e9b-4255-b889-671dd4a5c36b, remote_addr: 127.0.0.1:59607, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.443 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/149/db, method: GET, req_id: 66eb705b-2e9b-4255-b889-671dd4a5c36b, remote_addr: 127.0.0.1:59607, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.444 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/095.copy" to "/tmp/downstairs-zrMnlo6G/00/000/095.replace"
Sep 22 23:22:23.444 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.445 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/095.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.445 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/095"
Sep 22 23:22:23.445 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/095.db"
Sep 22 23:22:23.445 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.445 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/095.replace" to "/tmp/downstairs-zrMnlo6G/00/000/095.completed"
Sep 22 23:22:23.445 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.445 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.445 DEBG [0] It's time to notify for 474
Sep 22 23:22:23.445 INFO Completion from [0] id:474 status:true
Sep 22 23:22:23.445 INFO [475/752] Repair commands completed
Sep 22 23:22:23.445 INFO Pop front: ReconcileIO { id: ReconciliationId(475), op: ExtentReopen { repair_id: ReconciliationId(475), extent_id: 149 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.445 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.445 INFO [0] received reconcile message
Sep 22 23:22:23.445 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(475), op: ExtentReopen { repair_id: ReconciliationId(475), extent_id: 149 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.445 INFO [0] client ExtentReopen { repair_id: ReconciliationId(475), extent_id: 149 }
Sep 22 23:22:23.446 INFO [1] received reconcile message
Sep 22 23:22:23.446 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(475), op: ExtentReopen { repair_id: ReconciliationId(475), extent_id: 149 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.446 INFO [1] client ExtentReopen { repair_id: ReconciliationId(475), extent_id: 149 }
Sep 22 23:22:23.446 INFO [2] received reconcile message
Sep 22 23:22:23.446 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(475), op: ExtentReopen { repair_id: ReconciliationId(475), extent_id: 149 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.446 INFO [2] client ExtentReopen { repair_id: ReconciliationId(475), extent_id: 149 }
Sep 22 23:22:23.446 DEBG 475 Reopen extent 149
Sep 22 23:22:23.446 DEBG 475 Reopen extent 149
Sep 22 23:22:23.447 DEBG 475 Reopen extent 149
Sep 22 23:22:23.447 DEBG [2] It's time to notify for 475
Sep 22 23:22:23.448 INFO Completion from [2] id:475 status:true
Sep 22 23:22:23.448 INFO [476/752] Repair commands completed
Sep 22 23:22:23.448 INFO Pop front: ReconcileIO { id: ReconciliationId(476), op: ExtentFlush { repair_id: ReconciliationId(476), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.448 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.448 INFO [0] received reconcile message
Sep 22 23:22:23.448 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(476), op: ExtentFlush { repair_id: ReconciliationId(476), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.448 INFO [0] client ExtentFlush { repair_id: ReconciliationId(476), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.448 INFO [1] received reconcile message
Sep 22 23:22:23.448 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(476), op: ExtentFlush { repair_id: ReconciliationId(476), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.448 DEBG Read      :1087 deps:[JobId(1086)] res:true
Sep 22 23:22:23.448 INFO [1] client ExtentFlush { repair_id: ReconciliationId(476), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.448 INFO [2] received reconcile message
Sep 22 23:22:23.448 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(476), op: ExtentFlush { repair_id: ReconciliationId(476), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.448 INFO [2] client ExtentFlush { repair_id: ReconciliationId(476), extent_id: 161, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.448 DEBG 476 Flush extent 161 with f:2 g:2
Sep 22 23:22:23.448 DEBG Flush just extent 161 with f:2 and g:2
Sep 22 23:22:23.448 DEBG [1] It's time to notify for 476
Sep 22 23:22:23.448 INFO Completion from [1] id:476 status:true
Sep 22 23:22:23.448 INFO [477/752] Repair commands completed
Sep 22 23:22:23.448 INFO Pop front: ReconcileIO { id: ReconciliationId(477), op: ExtentClose { repair_id: ReconciliationId(477), extent_id: 161 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.448 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.448 INFO [0] received reconcile message
Sep 22 23:22:23.448 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(477), op: ExtentClose { repair_id: ReconciliationId(477), extent_id: 161 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.448 INFO [0] client ExtentClose { repair_id: ReconciliationId(477), extent_id: 161 }
Sep 22 23:22:23.448 INFO [1] received reconcile message
Sep 22 23:22:23.448 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(477), op: ExtentClose { repair_id: ReconciliationId(477), extent_id: 161 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.448 INFO [1] client ExtentClose { repair_id: ReconciliationId(477), extent_id: 161 }
Sep 22 23:22:23.448 INFO [2] received reconcile message
Sep 22 23:22:23.448 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(477), op: ExtentClose { repair_id: ReconciliationId(477), extent_id: 161 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.448 INFO [2] client ExtentClose { repair_id: ReconciliationId(477), extent_id: 161 }
Sep 22 23:22:23.449 DEBG 477 Close extent 161
Sep 22 23:22:23.449 DEBG 477 Close extent 161
Sep 22 23:22:23.449 DEBG 477 Close extent 161
Sep 22 23:22:23.449 DEBG [2] It's time to notify for 477
Sep 22 23:22:23.450 INFO Completion from [2] id:477 status:true
Sep 22 23:22:23.450 INFO [478/752] Repair commands completed
Sep 22 23:22:23.450 INFO Pop front: ReconcileIO { id: ReconciliationId(478), op: ExtentRepair { repair_id: ReconciliationId(478), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.450 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.450 INFO [0] received reconcile message
Sep 22 23:22:23.450 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(478), op: ExtentRepair { repair_id: ReconciliationId(478), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.450 INFO [0] client ExtentRepair { repair_id: ReconciliationId(478), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.450 INFO [0] Sending repair request ReconciliationId(478)
Sep 22 23:22:23.450 INFO [1] received reconcile message
Sep 22 23:22:23.450 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(478), op: ExtentRepair { repair_id: ReconciliationId(478), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.450 INFO [1] client ExtentRepair { repair_id: ReconciliationId(478), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.450 INFO [1] No action required ReconciliationId(478)
Sep 22 23:22:23.450 INFO [2] received reconcile message
Sep 22 23:22:23.450 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(478), op: ExtentRepair { repair_id: ReconciliationId(478), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.450 INFO [2] client ExtentRepair { repair_id: ReconciliationId(478), extent_id: 161, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.450 INFO [2] No action required ReconciliationId(478)
Sep 22 23:22:23.450 DEBG 478 Repair extent 161 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.450 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0A1.copy"
Sep 22 23:22:23.470 DEBG up_ds_listen was notified
Sep 22 23:22:23.470 DEBG up_ds_listen process 1086
Sep 22 23:22:23.470 DEBG [A] ack job 1086:87, : downstairs
Sep 22 23:22:23.470 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:23.472 WARN returning error on flush!
Sep 22 23:22:23.472 DEBG Flush     :1088 extent_limit None deps:[JobId(1087), JobId(1086)] res:false f:33 g:1
Sep 22 23:22:23.472 INFO [lossy] skipping 1088
Sep 22 23:22:23.472 INFO [lossy] skipping 1088
Sep 22 23:22:23.472 DEBG Flush     :1088 extent_limit None deps:[JobId(1087), JobId(1086)] res:true f:33 g:1
Sep 22 23:22:23.472 INFO [lossy] sleeping 1 second
Sep 22 23:22:23.515 INFO accepted connection, remote_addr: 127.0.0.1:47981, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.515 TRCE incoming request, uri: /extent/161/files, method: GET, req_id: 5fbaf268-0432-48df-bfeb-03e97c4425a2, remote_addr: 127.0.0.1:47981, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.515 INFO request completed, latency_us: 212, response_code: 200, uri: /extent/161/files, method: GET, req_id: 5fbaf268-0432-48df-bfeb-03e97c4425a2, remote_addr: 127.0.0.1:47981, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.516 INFO eid:161 Found repair files: ["0A1", "0A1.db"]
Sep 22 23:22:23.516 TRCE incoming request, uri: /newextent/161/data, method: GET, req_id: 20aee4b2-a557-4301-8c8c-adbedcea927b, remote_addr: 127.0.0.1:47981, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.516 INFO request completed, latency_us: 330, response_code: 200, uri: /newextent/161/data, method: GET, req_id: 20aee4b2-a557-4301-8c8c-adbedcea927b, remote_addr: 127.0.0.1:47981, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.521 TRCE incoming request, uri: /newextent/161/db, method: GET, req_id: 835d9d23-105d-494a-b302-a967e4efb1f1, remote_addr: 127.0.0.1:47981, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.521 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/161/db, method: GET, req_id: 835d9d23-105d-494a-b302-a967e4efb1f1, remote_addr: 127.0.0.1:47981, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.523 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0A1.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0A1.replace"
Sep 22 23:22:23.523 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.523 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0A1.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.524 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A1"
Sep 22 23:22:23.524 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A1.db"
Sep 22 23:22:23.524 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.524 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0A1.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0A1.completed"
Sep 22 23:22:23.524 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.524 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.524 DEBG [0] It's time to notify for 478
Sep 22 23:22:23.524 INFO Completion from [0] id:478 status:true
Sep 22 23:22:23.524 INFO [479/752] Repair commands completed
Sep 22 23:22:23.524 INFO Pop front: ReconcileIO { id: ReconciliationId(479), op: ExtentReopen { repair_id: ReconciliationId(479), extent_id: 161 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.524 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.524 INFO [0] received reconcile message
Sep 22 23:22:23.524 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(479), op: ExtentReopen { repair_id: ReconciliationId(479), extent_id: 161 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.524 INFO [0] client ExtentReopen { repair_id: ReconciliationId(479), extent_id: 161 }
Sep 22 23:22:23.524 INFO [1] received reconcile message
Sep 22 23:22:23.524 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(479), op: ExtentReopen { repair_id: ReconciliationId(479), extent_id: 161 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.524 INFO [1] client ExtentReopen { repair_id: ReconciliationId(479), extent_id: 161 }
Sep 22 23:22:23.525 INFO [2] received reconcile message
Sep 22 23:22:23.525 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(479), op: ExtentReopen { repair_id: ReconciliationId(479), extent_id: 161 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.525 INFO [2] client ExtentReopen { repair_id: ReconciliationId(479), extent_id: 161 }
Sep 22 23:22:23.525 DEBG 479 Reopen extent 161
Sep 22 23:22:23.525 DEBG 479 Reopen extent 161
Sep 22 23:22:23.526 DEBG 479 Reopen extent 161
Sep 22 23:22:23.526 DEBG [2] It's time to notify for 479
Sep 22 23:22:23.527 INFO Completion from [2] id:479 status:true
Sep 22 23:22:23.527 INFO [480/752] Repair commands completed
Sep 22 23:22:23.527 INFO Pop front: ReconcileIO { id: ReconciliationId(480), op: ExtentFlush { repair_id: ReconciliationId(480), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.527 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.527 INFO [0] received reconcile message
Sep 22 23:22:23.527 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(480), op: ExtentFlush { repair_id: ReconciliationId(480), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.527 INFO [0] client ExtentFlush { repair_id: ReconciliationId(480), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.527 INFO [1] received reconcile message
Sep 22 23:22:23.527 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(480), op: ExtentFlush { repair_id: ReconciliationId(480), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.527 INFO [1] client ExtentFlush { repair_id: ReconciliationId(480), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.527 INFO [2] received reconcile message
Sep 22 23:22:23.527 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(480), op: ExtentFlush { repair_id: ReconciliationId(480), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.527 INFO [2] client ExtentFlush { repair_id: ReconciliationId(480), extent_id: 71, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.527 DEBG 480 Flush extent 71 with f:2 g:2
Sep 22 23:22:23.527 DEBG Flush just extent 71 with f:2 and g:2
Sep 22 23:22:23.527 DEBG [1] It's time to notify for 480
Sep 22 23:22:23.527 INFO Completion from [1] id:480 status:true
Sep 22 23:22:23.527 INFO [481/752] Repair commands completed
Sep 22 23:22:23.527 INFO Pop front: ReconcileIO { id: ReconciliationId(481), op: ExtentClose { repair_id: ReconciliationId(481), extent_id: 71 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.527 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.527 INFO [0] received reconcile message
Sep 22 23:22:23.527 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(481), op: ExtentClose { repair_id: ReconciliationId(481), extent_id: 71 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.527 INFO [0] client ExtentClose { repair_id: ReconciliationId(481), extent_id: 71 }
Sep 22 23:22:23.527 INFO [1] received reconcile message
Sep 22 23:22:23.527 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(481), op: ExtentClose { repair_id: ReconciliationId(481), extent_id: 71 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.527 INFO [1] client ExtentClose { repair_id: ReconciliationId(481), extent_id: 71 }
Sep 22 23:22:23.527 INFO [2] received reconcile message
Sep 22 23:22:23.527 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(481), op: ExtentClose { repair_id: ReconciliationId(481), extent_id: 71 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.527 INFO [2] client ExtentClose { repair_id: ReconciliationId(481), extent_id: 71 }
Sep 22 23:22:23.528 DEBG 481 Close extent 71
Sep 22 23:22:23.528 DEBG 481 Close extent 71
Sep 22 23:22:23.528 DEBG 481 Close extent 71
Sep 22 23:22:23.529 DEBG [2] It's time to notify for 481
Sep 22 23:22:23.529 INFO Completion from [2] id:481 status:true
Sep 22 23:22:23.529 INFO [482/752] Repair commands completed
Sep 22 23:22:23.529 INFO Pop front: ReconcileIO { id: ReconciliationId(482), op: ExtentRepair { repair_id: ReconciliationId(482), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.529 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.529 INFO [0] received reconcile message
Sep 22 23:22:23.529 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(482), op: ExtentRepair { repair_id: ReconciliationId(482), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.529 INFO [0] client ExtentRepair { repair_id: ReconciliationId(482), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.529 INFO [0] Sending repair request ReconciliationId(482)
Sep 22 23:22:23.529 INFO [1] received reconcile message
Sep 22 23:22:23.529 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(482), op: ExtentRepair { repair_id: ReconciliationId(482), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.529 INFO [1] client ExtentRepair { repair_id: ReconciliationId(482), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.529 INFO [1] No action required ReconciliationId(482)
Sep 22 23:22:23.529 INFO [2] received reconcile message
Sep 22 23:22:23.529 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(482), op: ExtentRepair { repair_id: ReconciliationId(482), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.529 INFO [2] client ExtentRepair { repair_id: ReconciliationId(482), extent_id: 71, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.529 INFO [2] No action required ReconciliationId(482)
Sep 22 23:22:23.529 DEBG 482 Repair extent 71 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.529 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/047.copy"
Sep 22 23:22:23.594 INFO accepted connection, remote_addr: 127.0.0.1:44221, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.595 TRCE incoming request, uri: /extent/71/files, method: GET, req_id: e1fd86a5-91eb-42eb-9ada-e4eb94f37b9e, remote_addr: 127.0.0.1:44221, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.595 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/71/files, method: GET, req_id: e1fd86a5-91eb-42eb-9ada-e4eb94f37b9e, remote_addr: 127.0.0.1:44221, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.595 INFO eid:71 Found repair files: ["047", "047.db"]
Sep 22 23:22:23.595 TRCE incoming request, uri: /newextent/71/data, method: GET, req_id: a912a885-45fd-4956-be87-f454ec33c455, remote_addr: 127.0.0.1:44221, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.596 INFO request completed, latency_us: 305, response_code: 200, uri: /newextent/71/data, method: GET, req_id: a912a885-45fd-4956-be87-f454ec33c455, remote_addr: 127.0.0.1:44221, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.600 TRCE incoming request, uri: /newextent/71/db, method: GET, req_id: 8c7a52fe-72cc-4b2d-a00f-15b4c0e7a547, remote_addr: 127.0.0.1:44221, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.601 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/71/db, method: GET, req_id: 8c7a52fe-72cc-4b2d-a00f-15b4c0e7a547, remote_addr: 127.0.0.1:44221, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.602 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/047.copy" to "/tmp/downstairs-zrMnlo6G/00/000/047.replace"
Sep 22 23:22:23.602 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.603 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/047.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.603 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/047"
Sep 22 23:22:23.603 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/047.db"
Sep 22 23:22:23.603 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.603 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/047.replace" to "/tmp/downstairs-zrMnlo6G/00/000/047.completed"
Sep 22 23:22:23.603 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.603 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.603 DEBG [0] It's time to notify for 482
Sep 22 23:22:23.604 INFO Completion from [0] id:482 status:true
Sep 22 23:22:23.604 INFO [483/752] Repair commands completed
Sep 22 23:22:23.604 INFO Pop front: ReconcileIO { id: ReconciliationId(483), op: ExtentReopen { repair_id: ReconciliationId(483), extent_id: 71 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.604 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.604 INFO [0] received reconcile message
Sep 22 23:22:23.604 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(483), op: ExtentReopen { repair_id: ReconciliationId(483), extent_id: 71 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.604 INFO [0] client ExtentReopen { repair_id: ReconciliationId(483), extent_id: 71 }
Sep 22 23:22:23.604 INFO [1] received reconcile message
Sep 22 23:22:23.604 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(483), op: ExtentReopen { repair_id: ReconciliationId(483), extent_id: 71 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.604 INFO [1] client ExtentReopen { repair_id: ReconciliationId(483), extent_id: 71 }
Sep 22 23:22:23.604 INFO [2] received reconcile message
Sep 22 23:22:23.604 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(483), op: ExtentReopen { repair_id: ReconciliationId(483), extent_id: 71 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.604 INFO [2] client ExtentReopen { repair_id: ReconciliationId(483), extent_id: 71 }
Sep 22 23:22:23.604 DEBG 483 Reopen extent 71
Sep 22 23:22:23.605 DEBG 483 Reopen extent 71
Sep 22 23:22:23.605 DEBG 483 Reopen extent 71
Sep 22 23:22:23.606 DEBG [2] It's time to notify for 483
Sep 22 23:22:23.606 INFO Completion from [2] id:483 status:true
Sep 22 23:22:23.606 INFO [484/752] Repair commands completed
Sep 22 23:22:23.606 INFO Pop front: ReconcileIO { id: ReconciliationId(484), op: ExtentFlush { repair_id: ReconciliationId(484), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.606 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.606 INFO [0] received reconcile message
Sep 22 23:22:23.606 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(484), op: ExtentFlush { repair_id: ReconciliationId(484), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.606 INFO [0] client ExtentFlush { repair_id: ReconciliationId(484), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.606 INFO [1] received reconcile message
Sep 22 23:22:23.606 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(484), op: ExtentFlush { repair_id: ReconciliationId(484), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.606 INFO [1] client ExtentFlush { repair_id: ReconciliationId(484), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.606 INFO [2] received reconcile message
Sep 22 23:22:23.606 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(484), op: ExtentFlush { repair_id: ReconciliationId(484), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.606 INFO [2] client ExtentFlush { repair_id: ReconciliationId(484), extent_id: 5, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.606 DEBG 484 Flush extent 5 with f:2 g:2
Sep 22 23:22:23.606 DEBG Flush just extent 5 with f:2 and g:2
Sep 22 23:22:23.606 DEBG [1] It's time to notify for 484
Sep 22 23:22:23.606 INFO Completion from [1] id:484 status:true
Sep 22 23:22:23.606 INFO [485/752] Repair commands completed
Sep 22 23:22:23.606 INFO Pop front: ReconcileIO { id: ReconciliationId(485), op: ExtentClose { repair_id: ReconciliationId(485), extent_id: 5 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.606 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.606 INFO [0] received reconcile message
Sep 22 23:22:23.606 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(485), op: ExtentClose { repair_id: ReconciliationId(485), extent_id: 5 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.606 INFO [0] client ExtentClose { repair_id: ReconciliationId(485), extent_id: 5 }
Sep 22 23:22:23.606 INFO [1] received reconcile message
Sep 22 23:22:23.606 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(485), op: ExtentClose { repair_id: ReconciliationId(485), extent_id: 5 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.606 INFO [1] client ExtentClose { repair_id: ReconciliationId(485), extent_id: 5 }
Sep 22 23:22:23.606 INFO [2] received reconcile message
Sep 22 23:22:23.607 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(485), op: ExtentClose { repair_id: ReconciliationId(485), extent_id: 5 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.607 INFO [2] client ExtentClose { repair_id: ReconciliationId(485), extent_id: 5 }
Sep 22 23:22:23.607 DEBG 485 Close extent 5
Sep 22 23:22:23.607 DEBG 485 Close extent 5
Sep 22 23:22:23.607 DEBG 485 Close extent 5
Sep 22 23:22:23.608 DEBG [2] It's time to notify for 485
Sep 22 23:22:23.608 INFO Completion from [2] id:485 status:true
Sep 22 23:22:23.608 INFO [486/752] Repair commands completed
Sep 22 23:22:23.608 INFO Pop front: ReconcileIO { id: ReconciliationId(486), op: ExtentRepair { repair_id: ReconciliationId(486), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.608 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.608 INFO [0] received reconcile message
Sep 22 23:22:23.608 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(486), op: ExtentRepair { repair_id: ReconciliationId(486), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.608 INFO [0] client ExtentRepair { repair_id: ReconciliationId(486), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.608 INFO [0] Sending repair request ReconciliationId(486)
Sep 22 23:22:23.608 INFO [1] received reconcile message
Sep 22 23:22:23.608 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(486), op: ExtentRepair { repair_id: ReconciliationId(486), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.608 INFO [1] client ExtentRepair { repair_id: ReconciliationId(486), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.608 INFO [1] No action required ReconciliationId(486)
Sep 22 23:22:23.608 INFO [2] received reconcile message
Sep 22 23:22:23.608 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(486), op: ExtentRepair { repair_id: ReconciliationId(486), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.608 INFO [2] client ExtentRepair { repair_id: ReconciliationId(486), extent_id: 5, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.608 INFO [2] No action required ReconciliationId(486)
Sep 22 23:22:23.608 DEBG 486 Repair extent 5 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.608 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/005.copy"
Sep 22 23:22:23.672 INFO accepted connection, remote_addr: 127.0.0.1:33534, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.672 TRCE incoming request, uri: /extent/5/files, method: GET, req_id: e05d4c00-66e0-4df3-b3b1-8d07125abd88, remote_addr: 127.0.0.1:33534, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.672 INFO request completed, latency_us: 196, response_code: 200, uri: /extent/5/files, method: GET, req_id: e05d4c00-66e0-4df3-b3b1-8d07125abd88, remote_addr: 127.0.0.1:33534, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.673 INFO eid:5 Found repair files: ["005", "005.db"]
Sep 22 23:22:23.673 TRCE incoming request, uri: /newextent/5/data, method: GET, req_id: 45657933-a49f-416e-802c-bd038a986667, remote_addr: 127.0.0.1:33534, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.673 INFO request completed, latency_us: 321, response_code: 200, uri: /newextent/5/data, method: GET, req_id: 45657933-a49f-416e-802c-bd038a986667, remote_addr: 127.0.0.1:33534, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.678 TRCE incoming request, uri: /newextent/5/db, method: GET, req_id: 64fa2149-d80d-482e-abc7-cc6e1050d373, remote_addr: 127.0.0.1:33534, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.679 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/5/db, method: GET, req_id: 64fa2149-d80d-482e-abc7-cc6e1050d373, remote_addr: 127.0.0.1:33534, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.680 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/005.copy" to "/tmp/downstairs-zrMnlo6G/00/000/005.replace"
Sep 22 23:22:23.680 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.680 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/005.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.681 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/005"
Sep 22 23:22:23.681 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/005.db"
Sep 22 23:22:23.681 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.681 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/005.replace" to "/tmp/downstairs-zrMnlo6G/00/000/005.completed"
Sep 22 23:22:23.681 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.681 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.681 DEBG [0] It's time to notify for 486
Sep 22 23:22:23.681 INFO Completion from [0] id:486 status:true
Sep 22 23:22:23.681 INFO [487/752] Repair commands completed
Sep 22 23:22:23.681 INFO Pop front: ReconcileIO { id: ReconciliationId(487), op: ExtentReopen { repair_id: ReconciliationId(487), extent_id: 5 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.681 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.681 INFO [0] received reconcile message
Sep 22 23:22:23.681 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(487), op: ExtentReopen { repair_id: ReconciliationId(487), extent_id: 5 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.681 INFO [0] client ExtentReopen { repair_id: ReconciliationId(487), extent_id: 5 }
Sep 22 23:22:23.681 INFO [1] received reconcile message
Sep 22 23:22:23.681 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(487), op: ExtentReopen { repair_id: ReconciliationId(487), extent_id: 5 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.681 INFO [1] client ExtentReopen { repair_id: ReconciliationId(487), extent_id: 5 }
Sep 22 23:22:23.682 INFO [2] received reconcile message
Sep 22 23:22:23.682 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(487), op: ExtentReopen { repair_id: ReconciliationId(487), extent_id: 5 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.682 INFO [2] client ExtentReopen { repair_id: ReconciliationId(487), extent_id: 5 }
Sep 22 23:22:23.682 DEBG 487 Reopen extent 5
Sep 22 23:22:23.682 DEBG 487 Reopen extent 5
Sep 22 23:22:23.683 DEBG 487 Reopen extent 5
Sep 22 23:22:23.683 DEBG [2] It's time to notify for 487
Sep 22 23:22:23.683 INFO Completion from [2] id:487 status:true
Sep 22 23:22:23.683 INFO [488/752] Repair commands completed
Sep 22 23:22:23.683 INFO Pop front: ReconcileIO { id: ReconciliationId(488), op: ExtentFlush { repair_id: ReconciliationId(488), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.684 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.684 INFO [0] received reconcile message
Sep 22 23:22:23.684 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(488), op: ExtentFlush { repair_id: ReconciliationId(488), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.684 INFO [0] client ExtentFlush { repair_id: ReconciliationId(488), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.684 INFO [1] received reconcile message
Sep 22 23:22:23.684 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(488), op: ExtentFlush { repair_id: ReconciliationId(488), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.684 INFO [1] client ExtentFlush { repair_id: ReconciliationId(488), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.684 INFO [2] received reconcile message
Sep 22 23:22:23.684 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(488), op: ExtentFlush { repair_id: ReconciliationId(488), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.684 INFO [2] client ExtentFlush { repair_id: ReconciliationId(488), extent_id: 23, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.684 DEBG 488 Flush extent 23 with f:2 g:2
Sep 22 23:22:23.684 DEBG Flush just extent 23 with f:2 and g:2
Sep 22 23:22:23.684 DEBG [1] It's time to notify for 488
Sep 22 23:22:23.684 INFO Completion from [1] id:488 status:true
Sep 22 23:22:23.684 INFO [489/752] Repair commands completed
Sep 22 23:22:23.684 INFO Pop front: ReconcileIO { id: ReconciliationId(489), op: ExtentClose { repair_id: ReconciliationId(489), extent_id: 23 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.684 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.684 INFO [0] received reconcile message
Sep 22 23:22:23.684 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(489), op: ExtentClose { repair_id: ReconciliationId(489), extent_id: 23 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.684 INFO [0] client ExtentClose { repair_id: ReconciliationId(489), extent_id: 23 }
Sep 22 23:22:23.684 INFO [1] received reconcile message
Sep 22 23:22:23.684 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(489), op: ExtentClose { repair_id: ReconciliationId(489), extent_id: 23 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.684 INFO [1] client ExtentClose { repair_id: ReconciliationId(489), extent_id: 23 }
Sep 22 23:22:23.684 INFO [2] received reconcile message
Sep 22 23:22:23.684 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(489), op: ExtentClose { repair_id: ReconciliationId(489), extent_id: 23 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.684 INFO [2] client ExtentClose { repair_id: ReconciliationId(489), extent_id: 23 }
Sep 22 23:22:23.684 DEBG 489 Close extent 23
Sep 22 23:22:23.685 DEBG 489 Close extent 23
Sep 22 23:22:23.685 DEBG 489 Close extent 23
Sep 22 23:22:23.685 DEBG [2] It's time to notify for 489
Sep 22 23:22:23.685 INFO Completion from [2] id:489 status:true
Sep 22 23:22:23.685 INFO [490/752] Repair commands completed
Sep 22 23:22:23.685 INFO Pop front: ReconcileIO { id: ReconciliationId(490), op: ExtentRepair { repair_id: ReconciliationId(490), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.685 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.686 INFO [0] received reconcile message
Sep 22 23:22:23.686 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(490), op: ExtentRepair { repair_id: ReconciliationId(490), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.686 INFO [0] client ExtentRepair { repair_id: ReconciliationId(490), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.686 INFO [0] Sending repair request ReconciliationId(490)
Sep 22 23:22:23.686 INFO [1] received reconcile message
Sep 22 23:22:23.686 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(490), op: ExtentRepair { repair_id: ReconciliationId(490), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.686 INFO [1] client ExtentRepair { repair_id: ReconciliationId(490), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.686 INFO [1] No action required ReconciliationId(490)
Sep 22 23:22:23.686 INFO [2] received reconcile message
Sep 22 23:22:23.686 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(490), op: ExtentRepair { repair_id: ReconciliationId(490), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.686 INFO [2] client ExtentRepair { repair_id: ReconciliationId(490), extent_id: 23, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.686 INFO [2] No action required ReconciliationId(490)
Sep 22 23:22:23.686 DEBG 490 Repair extent 23 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.686 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/017.copy"
Sep 22 23:22:23.750 INFO accepted connection, remote_addr: 127.0.0.1:65203, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.750 TRCE incoming request, uri: /extent/23/files, method: GET, req_id: 95f04b47-842f-42e5-826c-c16c6d7218cf, remote_addr: 127.0.0.1:65203, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.751 INFO request completed, latency_us: 197, response_code: 200, uri: /extent/23/files, method: GET, req_id: 95f04b47-842f-42e5-826c-c16c6d7218cf, remote_addr: 127.0.0.1:65203, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.751 INFO eid:23 Found repair files: ["017", "017.db"]
Sep 22 23:22:23.751 TRCE incoming request, uri: /newextent/23/data, method: GET, req_id: 8f1ac1f2-06a0-4f5a-8548-ad5f0a952093, remote_addr: 127.0.0.1:65203, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.751 INFO request completed, latency_us: 322, response_code: 200, uri: /newextent/23/data, method: GET, req_id: 8f1ac1f2-06a0-4f5a-8548-ad5f0a952093, remote_addr: 127.0.0.1:65203, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.756 TRCE incoming request, uri: /newextent/23/db, method: GET, req_id: ddca5c4a-2afc-4921-a66b-3e0f64fa81f9, remote_addr: 127.0.0.1:65203, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.757 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/23/db, method: GET, req_id: ddca5c4a-2afc-4921-a66b-3e0f64fa81f9, remote_addr: 127.0.0.1:65203, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.758 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/017.copy" to "/tmp/downstairs-zrMnlo6G/00/000/017.replace"
Sep 22 23:22:23.758 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.758 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/017.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.759 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/017"
Sep 22 23:22:23.759 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/017.db"
Sep 22 23:22:23.759 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.759 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/017.replace" to "/tmp/downstairs-zrMnlo6G/00/000/017.completed"
Sep 22 23:22:23.759 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.759 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.759 DEBG [0] It's time to notify for 490
Sep 22 23:22:23.759 INFO Completion from [0] id:490 status:true
Sep 22 23:22:23.759 INFO [491/752] Repair commands completed
Sep 22 23:22:23.759 INFO Pop front: ReconcileIO { id: ReconciliationId(491), op: ExtentReopen { repair_id: ReconciliationId(491), extent_id: 23 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.759 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.759 INFO [0] received reconcile message
Sep 22 23:22:23.759 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(491), op: ExtentReopen { repair_id: ReconciliationId(491), extent_id: 23 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.759 INFO [0] client ExtentReopen { repair_id: ReconciliationId(491), extent_id: 23 }
Sep 22 23:22:23.759 INFO [1] received reconcile message
Sep 22 23:22:23.759 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(491), op: ExtentReopen { repair_id: ReconciliationId(491), extent_id: 23 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.759 INFO [1] client ExtentReopen { repair_id: ReconciliationId(491), extent_id: 23 }
Sep 22 23:22:23.760 INFO [2] received reconcile message
Sep 22 23:22:23.760 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(491), op: ExtentReopen { repair_id: ReconciliationId(491), extent_id: 23 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.760 INFO [2] client ExtentReopen { repair_id: ReconciliationId(491), extent_id: 23 }
Sep 22 23:22:23.760 DEBG 491 Reopen extent 23
Sep 22 23:22:23.760 DEBG 491 Reopen extent 23
Sep 22 23:22:23.761 DEBG 491 Reopen extent 23
Sep 22 23:22:23.761 DEBG [2] It's time to notify for 491
Sep 22 23:22:23.761 INFO Completion from [2] id:491 status:true
Sep 22 23:22:23.761 INFO [492/752] Repair commands completed
Sep 22 23:22:23.761 INFO Pop front: ReconcileIO { id: ReconciliationId(492), op: ExtentFlush { repair_id: ReconciliationId(492), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.761 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.761 INFO [0] received reconcile message
Sep 22 23:22:23.761 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(492), op: ExtentFlush { repair_id: ReconciliationId(492), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.762 INFO [0] client ExtentFlush { repair_id: ReconciliationId(492), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.762 INFO [1] received reconcile message
Sep 22 23:22:23.762 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(492), op: ExtentFlush { repair_id: ReconciliationId(492), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.762 INFO [1] client ExtentFlush { repair_id: ReconciliationId(492), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.762 INFO [2] received reconcile message
Sep 22 23:22:23.762 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(492), op: ExtentFlush { repair_id: ReconciliationId(492), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.762 INFO [2] client ExtentFlush { repair_id: ReconciliationId(492), extent_id: 98, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.762 DEBG 492 Flush extent 98 with f:2 g:2
Sep 22 23:22:23.762 DEBG Flush just extent 98 with f:2 and g:2
Sep 22 23:22:23.762 DEBG [1] It's time to notify for 492
Sep 22 23:22:23.762 INFO Completion from [1] id:492 status:true
Sep 22 23:22:23.762 INFO [493/752] Repair commands completed
Sep 22 23:22:23.762 INFO Pop front: ReconcileIO { id: ReconciliationId(493), op: ExtentClose { repair_id: ReconciliationId(493), extent_id: 98 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.762 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.762 INFO [0] received reconcile message
Sep 22 23:22:23.762 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(493), op: ExtentClose { repair_id: ReconciliationId(493), extent_id: 98 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.762 INFO [0] client ExtentClose { repair_id: ReconciliationId(493), extent_id: 98 }
Sep 22 23:22:23.762 INFO [1] received reconcile message
Sep 22 23:22:23.762 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(493), op: ExtentClose { repair_id: ReconciliationId(493), extent_id: 98 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.762 INFO [1] client ExtentClose { repair_id: ReconciliationId(493), extent_id: 98 }
Sep 22 23:22:23.762 INFO [2] received reconcile message
Sep 22 23:22:23.762 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(493), op: ExtentClose { repair_id: ReconciliationId(493), extent_id: 98 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.762 INFO [2] client ExtentClose { repair_id: ReconciliationId(493), extent_id: 98 }
Sep 22 23:22:23.762 DEBG 493 Close extent 98
Sep 22 23:22:23.763 DEBG 493 Close extent 98
Sep 22 23:22:23.763 DEBG 493 Close extent 98
Sep 22 23:22:23.763 DEBG [2] It's time to notify for 493
Sep 22 23:22:23.763 INFO Completion from [2] id:493 status:true
Sep 22 23:22:23.763 INFO [494/752] Repair commands completed
Sep 22 23:22:23.763 INFO Pop front: ReconcileIO { id: ReconciliationId(494), op: ExtentRepair { repair_id: ReconciliationId(494), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.763 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.763 INFO [0] received reconcile message
Sep 22 23:22:23.764 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(494), op: ExtentRepair { repair_id: ReconciliationId(494), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.764 INFO [0] client ExtentRepair { repair_id: ReconciliationId(494), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.764 INFO [0] Sending repair request ReconciliationId(494)
Sep 22 23:22:23.764 INFO [1] received reconcile message
Sep 22 23:22:23.764 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(494), op: ExtentRepair { repair_id: ReconciliationId(494), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.764 INFO [1] client ExtentRepair { repair_id: ReconciliationId(494), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.764 INFO [1] No action required ReconciliationId(494)
Sep 22 23:22:23.764 INFO [2] received reconcile message
Sep 22 23:22:23.764 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(494), op: ExtentRepair { repair_id: ReconciliationId(494), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.764 INFO [2] client ExtentRepair { repair_id: ReconciliationId(494), extent_id: 98, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.764 INFO [2] No action required ReconciliationId(494)
Sep 22 23:22:23.764 DEBG 494 Repair extent 98 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.764 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/062.copy"
Sep 22 23:22:23.825 INFO accepted connection, remote_addr: 127.0.0.1:49104, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.826 TRCE incoming request, uri: /extent/98/files, method: GET, req_id: de2be4e1-9ffb-464f-b9e2-0eb78e7c5949, remote_addr: 127.0.0.1:49104, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.826 INFO request completed, latency_us: 201, response_code: 200, uri: /extent/98/files, method: GET, req_id: de2be4e1-9ffb-464f-b9e2-0eb78e7c5949, remote_addr: 127.0.0.1:49104, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.826 INFO eid:98 Found repair files: ["062", "062.db"]
Sep 22 23:22:23.826 TRCE incoming request, uri: /newextent/98/data, method: GET, req_id: 1c007ddd-e9f5-49f8-bba8-e5211363203f, remote_addr: 127.0.0.1:49104, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.827 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/98/data, method: GET, req_id: 1c007ddd-e9f5-49f8-bba8-e5211363203f, remote_addr: 127.0.0.1:49104, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.832 TRCE incoming request, uri: /newextent/98/db, method: GET, req_id: a5e79f21-ce0b-4065-9420-ac9b1ccd1aa9, remote_addr: 127.0.0.1:49104, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.832 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/98/db, method: GET, req_id: a5e79f21-ce0b-4065-9420-ac9b1ccd1aa9, remote_addr: 127.0.0.1:49104, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.833 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/062.copy" to "/tmp/downstairs-zrMnlo6G/00/000/062.replace"
Sep 22 23:22:23.833 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.834 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/062.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.834 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/062"
Sep 22 23:22:23.834 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/062.db"
Sep 22 23:22:23.834 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.834 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/062.replace" to "/tmp/downstairs-zrMnlo6G/00/000/062.completed"
Sep 22 23:22:23.834 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.835 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.835 DEBG [0] It's time to notify for 494
Sep 22 23:22:23.835 INFO Completion from [0] id:494 status:true
Sep 22 23:22:23.835 INFO [495/752] Repair commands completed
Sep 22 23:22:23.835 INFO Pop front: ReconcileIO { id: ReconciliationId(495), op: ExtentReopen { repair_id: ReconciliationId(495), extent_id: 98 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.835 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.835 INFO [0] received reconcile message
Sep 22 23:22:23.835 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(495), op: ExtentReopen { repair_id: ReconciliationId(495), extent_id: 98 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.835 INFO [0] client ExtentReopen { repair_id: ReconciliationId(495), extent_id: 98 }
Sep 22 23:22:23.835 INFO [1] received reconcile message
Sep 22 23:22:23.835 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(495), op: ExtentReopen { repair_id: ReconciliationId(495), extent_id: 98 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.835 INFO [1] client ExtentReopen { repair_id: ReconciliationId(495), extent_id: 98 }
Sep 22 23:22:23.835 INFO [2] received reconcile message
Sep 22 23:22:23.835 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(495), op: ExtentReopen { repair_id: ReconciliationId(495), extent_id: 98 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.835 INFO [2] client ExtentReopen { repair_id: ReconciliationId(495), extent_id: 98 }
Sep 22 23:22:23.835 DEBG 495 Reopen extent 98
Sep 22 23:22:23.836 DEBG 495 Reopen extent 98
Sep 22 23:22:23.836 DEBG 495 Reopen extent 98
Sep 22 23:22:23.837 DEBG [2] It's time to notify for 495
Sep 22 23:22:23.837 INFO Completion from [2] id:495 status:true
Sep 22 23:22:23.837 INFO [496/752] Repair commands completed
Sep 22 23:22:23.837 INFO Pop front: ReconcileIO { id: ReconciliationId(496), op: ExtentFlush { repair_id: ReconciliationId(496), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.837 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.837 INFO [0] received reconcile message
Sep 22 23:22:23.837 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(496), op: ExtentFlush { repair_id: ReconciliationId(496), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.837 INFO [0] client ExtentFlush { repair_id: ReconciliationId(496), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.837 INFO [1] received reconcile message
Sep 22 23:22:23.837 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(496), op: ExtentFlush { repair_id: ReconciliationId(496), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.837 INFO [1] client ExtentFlush { repair_id: ReconciliationId(496), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.837 INFO [2] received reconcile message
Sep 22 23:22:23.837 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(496), op: ExtentFlush { repair_id: ReconciliationId(496), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.837 INFO [2] client ExtentFlush { repair_id: ReconciliationId(496), extent_id: 1, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.837 DEBG 496 Flush extent 1 with f:2 g:2
Sep 22 23:22:23.837 DEBG Flush just extent 1 with f:2 and g:2
Sep 22 23:22:23.838 DEBG [1] It's time to notify for 496
Sep 22 23:22:23.838 INFO Completion from [1] id:496 status:true
Sep 22 23:22:23.838 INFO [497/752] Repair commands completed
Sep 22 23:22:23.838 INFO Pop front: ReconcileIO { id: ReconciliationId(497), op: ExtentClose { repair_id: ReconciliationId(497), extent_id: 1 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.838 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.838 INFO [0] received reconcile message
Sep 22 23:22:23.838 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(497), op: ExtentClose { repair_id: ReconciliationId(497), extent_id: 1 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.838 INFO [0] client ExtentClose { repair_id: ReconciliationId(497), extent_id: 1 }
Sep 22 23:22:23.838 INFO [1] received reconcile message
Sep 22 23:22:23.838 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(497), op: ExtentClose { repair_id: ReconciliationId(497), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.838 INFO [1] client ExtentClose { repair_id: ReconciliationId(497), extent_id: 1 }
Sep 22 23:22:23.838 INFO [2] received reconcile message
Sep 22 23:22:23.838 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(497), op: ExtentClose { repair_id: ReconciliationId(497), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.838 INFO [2] client ExtentClose { repair_id: ReconciliationId(497), extent_id: 1 }
Sep 22 23:22:23.838 DEBG 497 Close extent 1
Sep 22 23:22:23.838 DEBG 497 Close extent 1
Sep 22 23:22:23.839 DEBG 497 Close extent 1
Sep 22 23:22:23.839 DEBG [2] It's time to notify for 497
Sep 22 23:22:23.839 INFO Completion from [2] id:497 status:true
Sep 22 23:22:23.839 INFO [498/752] Repair commands completed
Sep 22 23:22:23.839 INFO Pop front: ReconcileIO { id: ReconciliationId(498), op: ExtentRepair { repair_id: ReconciliationId(498), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.839 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.839 INFO [0] received reconcile message
Sep 22 23:22:23.839 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(498), op: ExtentRepair { repair_id: ReconciliationId(498), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.839 INFO [0] client ExtentRepair { repair_id: ReconciliationId(498), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.839 INFO [0] Sending repair request ReconciliationId(498)
Sep 22 23:22:23.839 INFO [1] received reconcile message
Sep 22 23:22:23.839 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(498), op: ExtentRepair { repair_id: ReconciliationId(498), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.839 INFO [1] client ExtentRepair { repair_id: ReconciliationId(498), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.839 INFO [1] No action required ReconciliationId(498)
Sep 22 23:22:23.839 INFO [2] received reconcile message
Sep 22 23:22:23.839 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(498), op: ExtentRepair { repair_id: ReconciliationId(498), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.839 INFO [2] client ExtentRepair { repair_id: ReconciliationId(498), extent_id: 1, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.839 INFO [2] No action required ReconciliationId(498)
Sep 22 23:22:23.839 DEBG 498 Repair extent 1 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.840 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/001.copy"
Sep 22 23:22:23.849 DEBG [1] Read AckReady 1087, : downstairs
Sep 22 23:22:23.850 ERRO [1] job id 1088 saw error GenericError("test error")
Sep 22 23:22:23.851 DEBG up_ds_listen was notified
Sep 22 23:22:23.851 DEBG up_ds_listen process 1087
Sep 22 23:22:23.851 DEBG [A] ack job 1087:88, : downstairs
Sep 22 23:22:23.904 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:23.904 INFO accepted connection, remote_addr: 127.0.0.1:58071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.904 TRCE incoming request, uri: /extent/1/files, method: GET, req_id: 371c8c00-324e-4a4e-8d74-c3bf7b0c1bfd, remote_addr: 127.0.0.1:58071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.904 INFO request completed, latency_us: 202, response_code: 200, uri: /extent/1/files, method: GET, req_id: 371c8c00-324e-4a4e-8d74-c3bf7b0c1bfd, remote_addr: 127.0.0.1:58071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.904 INFO eid:1 Found repair files: ["001", "001.db"]
Sep 22 23:22:23.905 TRCE incoming request, uri: /newextent/1/data, method: GET, req_id: a2cb3bd5-9998-4a8c-b07f-6f4f912d4866, remote_addr: 127.0.0.1:58071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.905 INFO request completed, latency_us: 261, response_code: 200, uri: /newextent/1/data, method: GET, req_id: a2cb3bd5-9998-4a8c-b07f-6f4f912d4866, remote_addr: 127.0.0.1:58071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.906 DEBG Flush     :1088 extent_limit None deps:[JobId(1087), JobId(1086)] res:true f:33 g:1
Sep 22 23:22:23.906 INFO [lossy] sleeping 1 second
Sep 22 23:22:23.906 WARN returning error on read!
Sep 22 23:22:23.906 DEBG Read      :1085 deps:[JobId(1084)] res:false
Sep 22 23:22:23.906 INFO [lossy] skipping 1087
Sep 22 23:22:23.906 WARN returning error on read!
Sep 22 23:22:23.906 DEBG Read      :1085 deps:[JobId(1084)] res:false
Sep 22 23:22:23.910 TRCE incoming request, uri: /newextent/1/db, method: GET, req_id: 64337320-c947-435f-84bd-b0d32d25e5f9, remote_addr: 127.0.0.1:58071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.911 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/1/db, method: GET, req_id: 64337320-c947-435f-84bd-b0d32d25e5f9, remote_addr: 127.0.0.1:58071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.912 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/001.copy" to "/tmp/downstairs-zrMnlo6G/00/000/001.replace"
Sep 22 23:22:23.912 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.912 DEBG Read      :1085 deps:[JobId(1084)] res:true
Sep 22 23:22:23.913 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/001.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.913 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/001"
Sep 22 23:22:23.913 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/001.db"
Sep 22 23:22:23.913 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.913 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/001.replace" to "/tmp/downstairs-zrMnlo6G/00/000/001.completed"
Sep 22 23:22:23.913 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.913 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.913 DEBG [0] It's time to notify for 498
Sep 22 23:22:23.913 INFO Completion from [0] id:498 status:true
Sep 22 23:22:23.913 INFO [499/752] Repair commands completed
Sep 22 23:22:23.913 INFO Pop front: ReconcileIO { id: ReconciliationId(499), op: ExtentReopen { repair_id: ReconciliationId(499), extent_id: 1 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.913 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.914 INFO [0] received reconcile message
Sep 22 23:22:23.914 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(499), op: ExtentReopen { repair_id: ReconciliationId(499), extent_id: 1 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.914 INFO [0] client ExtentReopen { repair_id: ReconciliationId(499), extent_id: 1 }
Sep 22 23:22:23.914 INFO [1] received reconcile message
Sep 22 23:22:23.914 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(499), op: ExtentReopen { repair_id: ReconciliationId(499), extent_id: 1 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.914 INFO [1] client ExtentReopen { repair_id: ReconciliationId(499), extent_id: 1 }
Sep 22 23:22:23.914 INFO [2] received reconcile message
Sep 22 23:22:23.914 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(499), op: ExtentReopen { repair_id: ReconciliationId(499), extent_id: 1 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.914 INFO [2] client ExtentReopen { repair_id: ReconciliationId(499), extent_id: 1 }
Sep 22 23:22:23.914 DEBG 499 Reopen extent 1
Sep 22 23:22:23.914 DEBG 499 Reopen extent 1
Sep 22 23:22:23.915 DEBG 499 Reopen extent 1
Sep 22 23:22:23.916 DEBG [2] It's time to notify for 499
Sep 22 23:22:23.916 INFO Completion from [2] id:499 status:true
Sep 22 23:22:23.916 INFO [500/752] Repair commands completed
Sep 22 23:22:23.916 INFO Pop front: ReconcileIO { id: ReconciliationId(500), op: ExtentFlush { repair_id: ReconciliationId(500), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.916 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.916 INFO [0] received reconcile message
Sep 22 23:22:23.916 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(500), op: ExtentFlush { repair_id: ReconciliationId(500), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.916 INFO [0] client ExtentFlush { repair_id: ReconciliationId(500), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.916 INFO [1] received reconcile message
Sep 22 23:22:23.916 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(500), op: ExtentFlush { repair_id: ReconciliationId(500), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.916 INFO [1] client ExtentFlush { repair_id: ReconciliationId(500), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.916 INFO [2] received reconcile message
Sep 22 23:22:23.916 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(500), op: ExtentFlush { repair_id: ReconciliationId(500), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.916 INFO [2] client ExtentFlush { repair_id: ReconciliationId(500), extent_id: 166, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.916 DEBG 500 Flush extent 166 with f:2 g:2
Sep 22 23:22:23.916 DEBG Flush just extent 166 with f:2 and g:2
Sep 22 23:22:23.916 DEBG [1] It's time to notify for 500
Sep 22 23:22:23.916 INFO Completion from [1] id:500 status:true
Sep 22 23:22:23.916 INFO [501/752] Repair commands completed
Sep 22 23:22:23.916 INFO Pop front: ReconcileIO { id: ReconciliationId(501), op: ExtentClose { repair_id: ReconciliationId(501), extent_id: 166 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.916 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.916 INFO [0] received reconcile message
Sep 22 23:22:23.916 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(501), op: ExtentClose { repair_id: ReconciliationId(501), extent_id: 166 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.916 INFO [0] client ExtentClose { repair_id: ReconciliationId(501), extent_id: 166 }
Sep 22 23:22:23.916 INFO [1] received reconcile message
Sep 22 23:22:23.916 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(501), op: ExtentClose { repair_id: ReconciliationId(501), extent_id: 166 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.916 INFO [1] client ExtentClose { repair_id: ReconciliationId(501), extent_id: 166 }
Sep 22 23:22:23.916 INFO [2] received reconcile message
Sep 22 23:22:23.917 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(501), op: ExtentClose { repair_id: ReconciliationId(501), extent_id: 166 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.917 INFO [2] client ExtentClose { repair_id: ReconciliationId(501), extent_id: 166 }
Sep 22 23:22:23.917 DEBG 501 Close extent 166
Sep 22 23:22:23.917 DEBG 501 Close extent 166
Sep 22 23:22:23.917 DEBG 501 Close extent 166
Sep 22 23:22:23.918 DEBG [2] It's time to notify for 501
Sep 22 23:22:23.918 INFO Completion from [2] id:501 status:true
Sep 22 23:22:23.918 INFO [502/752] Repair commands completed
Sep 22 23:22:23.918 INFO Pop front: ReconcileIO { id: ReconciliationId(502), op: ExtentRepair { repair_id: ReconciliationId(502), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.918 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.918 INFO [0] received reconcile message
Sep 22 23:22:23.918 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(502), op: ExtentRepair { repair_id: ReconciliationId(502), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.918 INFO [0] client ExtentRepair { repair_id: ReconciliationId(502), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.918 INFO [0] Sending repair request ReconciliationId(502)
Sep 22 23:22:23.918 INFO [1] received reconcile message
Sep 22 23:22:23.918 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(502), op: ExtentRepair { repair_id: ReconciliationId(502), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.918 INFO [1] client ExtentRepair { repair_id: ReconciliationId(502), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.918 INFO [1] No action required ReconciliationId(502)
Sep 22 23:22:23.918 INFO [2] received reconcile message
Sep 22 23:22:23.918 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(502), op: ExtentRepair { repair_id: ReconciliationId(502), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.918 INFO [2] client ExtentRepair { repair_id: ReconciliationId(502), extent_id: 166, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.918 INFO [2] No action required ReconciliationId(502)
Sep 22 23:22:23.918 DEBG 502 Repair extent 166 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.918 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0A6.copy"
Sep 22 23:22:23.981 INFO accepted connection, remote_addr: 127.0.0.1:54799, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.982 TRCE incoming request, uri: /extent/166/files, method: GET, req_id: e1e93694-168c-42b9-89a1-097cf9321150, remote_addr: 127.0.0.1:54799, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.982 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/166/files, method: GET, req_id: e1e93694-168c-42b9-89a1-097cf9321150, remote_addr: 127.0.0.1:54799, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.982 INFO eid:166 Found repair files: ["0A6", "0A6.db"]
Sep 22 23:22:23.982 DEBG IO Read  1089 has deps [JobId(1088)]
Sep 22 23:22:23.982 TRCE incoming request, uri: /newextent/166/data, method: GET, req_id: 0f3b0f27-38b7-4baf-8f1c-7120939c0811, remote_addr: 127.0.0.1:54799, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.983 INFO request completed, latency_us: 308, response_code: 200, uri: /newextent/166/data, method: GET, req_id: 0f3b0f27-38b7-4baf-8f1c-7120939c0811, remote_addr: 127.0.0.1:54799, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.987 TRCE incoming request, uri: /newextent/166/db, method: GET, req_id: 10f416ee-7917-4c50-8eea-76ec84ec4913, remote_addr: 127.0.0.1:54799, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.987 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/166/db, method: GET, req_id: 10f416ee-7917-4c50-8eea-76ec84ec4913, remote_addr: 127.0.0.1:54799, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:23.989 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0A6.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0A6.replace"
Sep 22 23:22:23.989 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.989 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0A6.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.990 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A6"
Sep 22 23:22:23.990 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A6.db"
Sep 22 23:22:23.990 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.990 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0A6.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0A6.completed"
Sep 22 23:22:23.990 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.990 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:23.990 DEBG [0] It's time to notify for 502
Sep 22 23:22:23.990 INFO Completion from [0] id:502 status:true
Sep 22 23:22:23.990 INFO [503/752] Repair commands completed
Sep 22 23:22:23.990 INFO Pop front: ReconcileIO { id: ReconciliationId(503), op: ExtentReopen { repair_id: ReconciliationId(503), extent_id: 166 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.990 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.990 INFO [0] received reconcile message
Sep 22 23:22:23.990 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(503), op: ExtentReopen { repair_id: ReconciliationId(503), extent_id: 166 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.990 INFO [0] client ExtentReopen { repair_id: ReconciliationId(503), extent_id: 166 }
Sep 22 23:22:23.990 INFO [1] received reconcile message
Sep 22 23:22:23.990 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(503), op: ExtentReopen { repair_id: ReconciliationId(503), extent_id: 166 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.990 INFO [1] client ExtentReopen { repair_id: ReconciliationId(503), extent_id: 166 }
Sep 22 23:22:23.990 INFO [2] received reconcile message
Sep 22 23:22:23.990 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(503), op: ExtentReopen { repair_id: ReconciliationId(503), extent_id: 166 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.990 INFO [2] client ExtentReopen { repair_id: ReconciliationId(503), extent_id: 166 }
Sep 22 23:22:23.991 DEBG 503 Reopen extent 166
Sep 22 23:22:23.991 DEBG 503 Reopen extent 166
Sep 22 23:22:23.992 DEBG 503 Reopen extent 166
Sep 22 23:22:23.992 DEBG [2] It's time to notify for 503
Sep 22 23:22:23.992 INFO Completion from [2] id:503 status:true
Sep 22 23:22:23.992 INFO [504/752] Repair commands completed
Sep 22 23:22:23.992 INFO Pop front: ReconcileIO { id: ReconciliationId(504), op: ExtentFlush { repair_id: ReconciliationId(504), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.992 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.992 INFO [0] received reconcile message
Sep 22 23:22:23.992 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(504), op: ExtentFlush { repair_id: ReconciliationId(504), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.992 INFO [0] client ExtentFlush { repair_id: ReconciliationId(504), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.993 INFO [1] received reconcile message
Sep 22 23:22:23.993 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(504), op: ExtentFlush { repair_id: ReconciliationId(504), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:23.993 INFO [1] client ExtentFlush { repair_id: ReconciliationId(504), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.993 INFO [2] received reconcile message
Sep 22 23:22:23.993 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(504), op: ExtentFlush { repair_id: ReconciliationId(504), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.993 INFO [2] client ExtentFlush { repair_id: ReconciliationId(504), extent_id: 185, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:23.993 DEBG 504 Flush extent 185 with f:2 g:2
Sep 22 23:22:23.993 DEBG Flush just extent 185 with f:2 and g:2
Sep 22 23:22:23.993 DEBG [1] It's time to notify for 504
Sep 22 23:22:23.993 INFO Completion from [1] id:504 status:true
Sep 22 23:22:23.993 INFO [505/752] Repair commands completed
Sep 22 23:22:23.993 INFO Pop front: ReconcileIO { id: ReconciliationId(505), op: ExtentClose { repair_id: ReconciliationId(505), extent_id: 185 }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.993 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.993 INFO [0] received reconcile message
Sep 22 23:22:23.993 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(505), op: ExtentClose { repair_id: ReconciliationId(505), extent_id: 185 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.993 INFO [0] client ExtentClose { repair_id: ReconciliationId(505), extent_id: 185 }
Sep 22 23:22:23.993 INFO [1] received reconcile message
Sep 22 23:22:23.993 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(505), op: ExtentClose { repair_id: ReconciliationId(505), extent_id: 185 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.993 INFO [1] client ExtentClose { repair_id: ReconciliationId(505), extent_id: 185 }
Sep 22 23:22:23.993 INFO [2] received reconcile message
Sep 22 23:22:23.993 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(505), op: ExtentClose { repair_id: ReconciliationId(505), extent_id: 185 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:23.993 INFO [2] client ExtentClose { repair_id: ReconciliationId(505), extent_id: 185 }
Sep 22 23:22:23.993 DEBG 505 Close extent 185
Sep 22 23:22:23.994 DEBG 505 Close extent 185
Sep 22 23:22:23.994 DEBG 505 Close extent 185
Sep 22 23:22:23.994 DEBG [2] It's time to notify for 505
Sep 22 23:22:23.994 INFO Completion from [2] id:505 status:true
Sep 22 23:22:23.994 INFO [506/752] Repair commands completed
Sep 22 23:22:23.994 INFO Pop front: ReconcileIO { id: ReconciliationId(506), op: ExtentRepair { repair_id: ReconciliationId(506), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:23.994 INFO Sent repair work, now wait for resp
Sep 22 23:22:23.994 INFO [0] received reconcile message
Sep 22 23:22:23.994 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(506), op: ExtentRepair { repair_id: ReconciliationId(506), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:23.994 INFO [0] client ExtentRepair { repair_id: ReconciliationId(506), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.994 INFO [0] Sending repair request ReconciliationId(506)
Sep 22 23:22:23.995 INFO [1] received reconcile message
Sep 22 23:22:23.995 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(506), op: ExtentRepair { repair_id: ReconciliationId(506), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:23.995 INFO [1] client ExtentRepair { repair_id: ReconciliationId(506), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.995 INFO [1] No action required ReconciliationId(506)
Sep 22 23:22:23.995 INFO [2] received reconcile message
Sep 22 23:22:23.995 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(506), op: ExtentRepair { repair_id: ReconciliationId(506), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:23.995 INFO [2] client ExtentRepair { repair_id: ReconciliationId(506), extent_id: 185, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:23.995 INFO [2] No action required ReconciliationId(506)
Sep 22 23:22:23.995 DEBG 506 Repair extent 185 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:23.995 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0B9.copy"
Sep 22 23:22:24.061 INFO accepted connection, remote_addr: 127.0.0.1:59376, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.061 TRCE incoming request, uri: /extent/185/files, method: GET, req_id: 79df7ec7-655d-4607-8f43-b981edf49c45, remote_addr: 127.0.0.1:59376, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.061 INFO request completed, latency_us: 249, response_code: 200, uri: /extent/185/files, method: GET, req_id: 79df7ec7-655d-4607-8f43-b981edf49c45, remote_addr: 127.0.0.1:59376, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.061 INFO eid:185 Found repair files: ["0B9", "0B9.db"]
Sep 22 23:22:24.062 TRCE incoming request, uri: /newextent/185/data, method: GET, req_id: 186dfc66-2023-4f50-bc1d-0a358b585f8b, remote_addr: 127.0.0.1:59376, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.062 INFO request completed, latency_us: 352, response_code: 200, uri: /newextent/185/data, method: GET, req_id: 186dfc66-2023-4f50-bc1d-0a358b585f8b, remote_addr: 127.0.0.1:59376, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.067 TRCE incoming request, uri: /newextent/185/db, method: GET, req_id: c5e5bc20-675e-4d5c-a85f-1a882baa737c, remote_addr: 127.0.0.1:59376, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.067 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/185/db, method: GET, req_id: c5e5bc20-675e-4d5c-a85f-1a882baa737c, remote_addr: 127.0.0.1:59376, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.068 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0B9.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0B9.replace"
Sep 22 23:22:24.068 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.069 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0B9.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.069 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B9"
Sep 22 23:22:24.069 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B9.db"
Sep 22 23:22:24.069 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.069 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0B9.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0B9.completed"
Sep 22 23:22:24.069 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.070 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.070 DEBG [0] It's time to notify for 506
Sep 22 23:22:24.070 INFO Completion from [0] id:506 status:true
Sep 22 23:22:24.070 INFO [507/752] Repair commands completed
Sep 22 23:22:24.070 INFO Pop front: ReconcileIO { id: ReconciliationId(507), op: ExtentReopen { repair_id: ReconciliationId(507), extent_id: 185 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.070 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.070 INFO [0] received reconcile message
Sep 22 23:22:24.070 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(507), op: ExtentReopen { repair_id: ReconciliationId(507), extent_id: 185 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.070 INFO [0] client ExtentReopen { repair_id: ReconciliationId(507), extent_id: 185 }
Sep 22 23:22:24.070 INFO [1] received reconcile message
Sep 22 23:22:24.070 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(507), op: ExtentReopen { repair_id: ReconciliationId(507), extent_id: 185 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.070 INFO [1] client ExtentReopen { repair_id: ReconciliationId(507), extent_id: 185 }
Sep 22 23:22:24.070 INFO [2] received reconcile message
Sep 22 23:22:24.070 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(507), op: ExtentReopen { repair_id: ReconciliationId(507), extent_id: 185 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.070 INFO [2] client ExtentReopen { repair_id: ReconciliationId(507), extent_id: 185 }
Sep 22 23:22:24.070 DEBG 507 Reopen extent 185
Sep 22 23:22:24.071 DEBG 507 Reopen extent 185
Sep 22 23:22:24.072 DEBG 507 Reopen extent 185
Sep 22 23:22:24.072 DEBG [2] It's time to notify for 507
Sep 22 23:22:24.072 INFO Completion from [2] id:507 status:true
Sep 22 23:22:24.072 INFO [508/752] Repair commands completed
Sep 22 23:22:24.072 INFO Pop front: ReconcileIO { id: ReconciliationId(508), op: ExtentFlush { repair_id: ReconciliationId(508), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.072 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.072 INFO [0] received reconcile message
Sep 22 23:22:24.072 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(508), op: ExtentFlush { repair_id: ReconciliationId(508), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.072 INFO [0] client ExtentFlush { repair_id: ReconciliationId(508), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.072 INFO [1] received reconcile message
Sep 22 23:22:24.072 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(508), op: ExtentFlush { repair_id: ReconciliationId(508), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.073 INFO [1] client ExtentFlush { repair_id: ReconciliationId(508), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.073 INFO [2] received reconcile message
Sep 22 23:22:24.073 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(508), op: ExtentFlush { repair_id: ReconciliationId(508), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.073 INFO [2] client ExtentFlush { repair_id: ReconciliationId(508), extent_id: 86, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.073 DEBG 508 Flush extent 86 with f:2 g:2
Sep 22 23:22:24.073 DEBG Flush just extent 86 with f:2 and g:2
Sep 22 23:22:24.073 DEBG [1] It's time to notify for 508
Sep 22 23:22:24.073 INFO Completion from [1] id:508 status:true
Sep 22 23:22:24.073 INFO [509/752] Repair commands completed
Sep 22 23:22:24.073 INFO Pop front: ReconcileIO { id: ReconciliationId(509), op: ExtentClose { repair_id: ReconciliationId(509), extent_id: 86 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.073 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.073 INFO [0] received reconcile message
Sep 22 23:22:24.073 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(509), op: ExtentClose { repair_id: ReconciliationId(509), extent_id: 86 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.073 INFO [0] client ExtentClose { repair_id: ReconciliationId(509), extent_id: 86 }
Sep 22 23:22:24.073 INFO [1] received reconcile message
Sep 22 23:22:24.073 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(509), op: ExtentClose { repair_id: ReconciliationId(509), extent_id: 86 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.073 INFO [1] client ExtentClose { repair_id: ReconciliationId(509), extent_id: 86 }
Sep 22 23:22:24.073 INFO [2] received reconcile message
Sep 22 23:22:24.073 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(509), op: ExtentClose { repair_id: ReconciliationId(509), extent_id: 86 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.073 INFO [2] client ExtentClose { repair_id: ReconciliationId(509), extent_id: 86 }
Sep 22 23:22:24.073 DEBG 509 Close extent 86
Sep 22 23:22:24.074 DEBG 509 Close extent 86
Sep 22 23:22:24.074 DEBG 509 Close extent 86
Sep 22 23:22:24.074 DEBG [2] It's time to notify for 509
Sep 22 23:22:24.074 INFO Completion from [2] id:509 status:true
Sep 22 23:22:24.074 INFO [510/752] Repair commands completed
Sep 22 23:22:24.074 INFO Pop front: ReconcileIO { id: ReconciliationId(510), op: ExtentRepair { repair_id: ReconciliationId(510), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.074 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.074 INFO [0] received reconcile message
Sep 22 23:22:24.074 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(510), op: ExtentRepair { repair_id: ReconciliationId(510), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.074 INFO [0] client ExtentRepair { repair_id: ReconciliationId(510), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.074 INFO [0] Sending repair request ReconciliationId(510)
Sep 22 23:22:24.075 INFO [1] received reconcile message
Sep 22 23:22:24.075 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(510), op: ExtentRepair { repair_id: ReconciliationId(510), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.075 INFO [1] client ExtentRepair { repair_id: ReconciliationId(510), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.075 INFO [1] No action required ReconciliationId(510)
Sep 22 23:22:24.075 INFO [2] received reconcile message
Sep 22 23:22:24.075 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(510), op: ExtentRepair { repair_id: ReconciliationId(510), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.075 INFO [2] client ExtentRepair { repair_id: ReconciliationId(510), extent_id: 86, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.075 INFO [2] No action required ReconciliationId(510)
Sep 22 23:22:24.075 DEBG 510 Repair extent 86 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.075 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/056.copy"
Sep 22 23:22:24.138 INFO accepted connection, remote_addr: 127.0.0.1:41753, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.139 TRCE incoming request, uri: /extent/86/files, method: GET, req_id: ebe66f86-cf9a-4324-9ba1-07a32bc89b64, remote_addr: 127.0.0.1:41753, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.139 INFO request completed, latency_us: 207, response_code: 200, uri: /extent/86/files, method: GET, req_id: ebe66f86-cf9a-4324-9ba1-07a32bc89b64, remote_addr: 127.0.0.1:41753, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.139 INFO eid:86 Found repair files: ["056", "056.db"]
Sep 22 23:22:24.139 TRCE incoming request, uri: /newextent/86/data, method: GET, req_id: 77642855-a184-4b7d-b8c0-6cbb8ee3bd46, remote_addr: 127.0.0.1:41753, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.140 INFO request completed, latency_us: 317, response_code: 200, uri: /newextent/86/data, method: GET, req_id: 77642855-a184-4b7d-b8c0-6cbb8ee3bd46, remote_addr: 127.0.0.1:41753, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.145 TRCE incoming request, uri: /newextent/86/db, method: GET, req_id: 749d84a4-1e3c-42b8-8a13-e650609ea139, remote_addr: 127.0.0.1:41753, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.145 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/86/db, method: GET, req_id: 749d84a4-1e3c-42b8-8a13-e650609ea139, remote_addr: 127.0.0.1:41753, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.146 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/056.copy" to "/tmp/downstairs-zrMnlo6G/00/000/056.replace"
Sep 22 23:22:24.146 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.147 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/056.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.147 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/056"
Sep 22 23:22:24.147 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/056.db"
Sep 22 23:22:24.147 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.147 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/056.replace" to "/tmp/downstairs-zrMnlo6G/00/000/056.completed"
Sep 22 23:22:24.147 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.147 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.147 DEBG [0] It's time to notify for 510
Sep 22 23:22:24.148 INFO Completion from [0] id:510 status:true
Sep 22 23:22:24.148 INFO [511/752] Repair commands completed
Sep 22 23:22:24.148 INFO Pop front: ReconcileIO { id: ReconciliationId(511), op: ExtentReopen { repair_id: ReconciliationId(511), extent_id: 86 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.148 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.148 INFO [0] received reconcile message
Sep 22 23:22:24.148 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(511), op: ExtentReopen { repair_id: ReconciliationId(511), extent_id: 86 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.148 INFO [0] client ExtentReopen { repair_id: ReconciliationId(511), extent_id: 86 }
Sep 22 23:22:24.148 INFO [1] received reconcile message
Sep 22 23:22:24.148 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(511), op: ExtentReopen { repair_id: ReconciliationId(511), extent_id: 86 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.148 INFO [1] client ExtentReopen { repair_id: ReconciliationId(511), extent_id: 86 }
Sep 22 23:22:24.148 INFO [2] received reconcile message
Sep 22 23:22:24.148 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(511), op: ExtentReopen { repair_id: ReconciliationId(511), extent_id: 86 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.148 INFO [2] client ExtentReopen { repair_id: ReconciliationId(511), extent_id: 86 }
Sep 22 23:22:24.148 DEBG 511 Reopen extent 86
Sep 22 23:22:24.149 DEBG 511 Reopen extent 86
Sep 22 23:22:24.149 DEBG 511 Reopen extent 86
Sep 22 23:22:24.150 DEBG [2] It's time to notify for 511
Sep 22 23:22:24.150 INFO Completion from [2] id:511 status:true
Sep 22 23:22:24.150 INFO [512/752] Repair commands completed
Sep 22 23:22:24.150 INFO Pop front: ReconcileIO { id: ReconciliationId(512), op: ExtentFlush { repair_id: ReconciliationId(512), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.150 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.150 INFO [0] received reconcile message
Sep 22 23:22:24.150 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(512), op: ExtentFlush { repair_id: ReconciliationId(512), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.150 INFO [0] client ExtentFlush { repair_id: ReconciliationId(512), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.150 INFO [1] received reconcile message
Sep 22 23:22:24.150 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(512), op: ExtentFlush { repair_id: ReconciliationId(512), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.150 INFO [1] client ExtentFlush { repair_id: ReconciliationId(512), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.150 INFO [2] received reconcile message
Sep 22 23:22:24.150 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(512), op: ExtentFlush { repair_id: ReconciliationId(512), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.150 INFO [2] client ExtentFlush { repair_id: ReconciliationId(512), extent_id: 135, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.150 DEBG 512 Flush extent 135 with f:2 g:2
Sep 22 23:22:24.150 DEBG Flush just extent 135 with f:2 and g:2
Sep 22 23:22:24.150 DEBG [1] It's time to notify for 512
Sep 22 23:22:24.150 INFO Completion from [1] id:512 status:true
Sep 22 23:22:24.150 INFO [513/752] Repair commands completed
Sep 22 23:22:24.150 INFO Pop front: ReconcileIO { id: ReconciliationId(513), op: ExtentClose { repair_id: ReconciliationId(513), extent_id: 135 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.150 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.151 INFO [0] received reconcile message
Sep 22 23:22:24.151 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(513), op: ExtentClose { repair_id: ReconciliationId(513), extent_id: 135 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.151 INFO [0] client ExtentClose { repair_id: ReconciliationId(513), extent_id: 135 }
Sep 22 23:22:24.151 INFO [1] received reconcile message
Sep 22 23:22:24.151 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(513), op: ExtentClose { repair_id: ReconciliationId(513), extent_id: 135 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.151 INFO [1] client ExtentClose { repair_id: ReconciliationId(513), extent_id: 135 }
Sep 22 23:22:24.151 INFO [2] received reconcile message
Sep 22 23:22:24.151 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(513), op: ExtentClose { repair_id: ReconciliationId(513), extent_id: 135 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.151 INFO [2] client ExtentClose { repair_id: ReconciliationId(513), extent_id: 135 }
Sep 22 23:22:24.151 DEBG 513 Close extent 135
Sep 22 23:22:24.151 DEBG 513 Close extent 135
Sep 22 23:22:24.151 DEBG 513 Close extent 135
Sep 22 23:22:24.152 DEBG [2] It's time to notify for 513
Sep 22 23:22:24.152 INFO Completion from [2] id:513 status:true
Sep 22 23:22:24.152 INFO [514/752] Repair commands completed
Sep 22 23:22:24.152 INFO Pop front: ReconcileIO { id: ReconciliationId(514), op: ExtentRepair { repair_id: ReconciliationId(514), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.152 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.152 INFO [0] received reconcile message
Sep 22 23:22:24.152 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(514), op: ExtentRepair { repair_id: ReconciliationId(514), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.152 INFO [0] client ExtentRepair { repair_id: ReconciliationId(514), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.152 INFO [0] Sending repair request ReconciliationId(514)
Sep 22 23:22:24.152 INFO [1] received reconcile message
Sep 22 23:22:24.152 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(514), op: ExtentRepair { repair_id: ReconciliationId(514), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.152 INFO [1] client ExtentRepair { repair_id: ReconciliationId(514), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.152 INFO [1] No action required ReconciliationId(514)
Sep 22 23:22:24.152 INFO [2] received reconcile message
Sep 22 23:22:24.152 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(514), op: ExtentRepair { repair_id: ReconciliationId(514), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.152 INFO [2] client ExtentRepair { repair_id: ReconciliationId(514), extent_id: 135, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.152 INFO [2] No action required ReconciliationId(514)
Sep 22 23:22:24.152 DEBG 514 Repair extent 135 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.152 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/087.copy"
Sep 22 23:22:24.217 INFO accepted connection, remote_addr: 127.0.0.1:45769, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.217 TRCE incoming request, uri: /extent/135/files, method: GET, req_id: 47f8941a-a98a-4e7b-844b-c2ace649dba0, remote_addr: 127.0.0.1:45769, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.217 INFO request completed, latency_us: 220, response_code: 200, uri: /extent/135/files, method: GET, req_id: 47f8941a-a98a-4e7b-844b-c2ace649dba0, remote_addr: 127.0.0.1:45769, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.217 INFO eid:135 Found repair files: ["087", "087.db"]
Sep 22 23:22:24.218 TRCE incoming request, uri: /newextent/135/data, method: GET, req_id: 05ec7851-46bd-4448-8e7f-599ec16aec8e, remote_addr: 127.0.0.1:45769, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.218 INFO request completed, latency_us: 266, response_code: 200, uri: /newextent/135/data, method: GET, req_id: 05ec7851-46bd-4448-8e7f-599ec16aec8e, remote_addr: 127.0.0.1:45769, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.223 TRCE incoming request, uri: /newextent/135/db, method: GET, req_id: 89bb2f99-4483-456e-abf4-684cde383111, remote_addr: 127.0.0.1:45769, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.223 INFO request completed, latency_us: 273, response_code: 200, uri: /newextent/135/db, method: GET, req_id: 89bb2f99-4483-456e-abf4-684cde383111, remote_addr: 127.0.0.1:45769, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.225 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/087.copy" to "/tmp/downstairs-zrMnlo6G/00/000/087.replace"
Sep 22 23:22:24.225 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.226 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/087.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.226 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/087"
Sep 22 23:22:24.226 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/087.db"
Sep 22 23:22:24.226 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.226 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/087.replace" to "/tmp/downstairs-zrMnlo6G/00/000/087.completed"
Sep 22 23:22:24.226 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.226 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.226 DEBG [0] It's time to notify for 514
Sep 22 23:22:24.226 INFO Completion from [0] id:514 status:true
Sep 22 23:22:24.226 INFO [515/752] Repair commands completed
Sep 22 23:22:24.226 INFO Pop front: ReconcileIO { id: ReconciliationId(515), op: ExtentReopen { repair_id: ReconciliationId(515), extent_id: 135 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.227 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.227 INFO [0] received reconcile message
Sep 22 23:22:24.227 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(515), op: ExtentReopen { repair_id: ReconciliationId(515), extent_id: 135 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.227 INFO [0] client ExtentReopen { repair_id: ReconciliationId(515), extent_id: 135 }
Sep 22 23:22:24.227 INFO [1] received reconcile message
Sep 22 23:22:24.227 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(515), op: ExtentReopen { repair_id: ReconciliationId(515), extent_id: 135 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.227 INFO [1] client ExtentReopen { repair_id: ReconciliationId(515), extent_id: 135 }
Sep 22 23:22:24.227 INFO [2] received reconcile message
Sep 22 23:22:24.227 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(515), op: ExtentReopen { repair_id: ReconciliationId(515), extent_id: 135 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.227 INFO [2] client ExtentReopen { repair_id: ReconciliationId(515), extent_id: 135 }
Sep 22 23:22:24.227 DEBG 515 Reopen extent 135
Sep 22 23:22:24.228 DEBG 515 Reopen extent 135
Sep 22 23:22:24.228 DEBG 515 Reopen extent 135
Sep 22 23:22:24.229 DEBG [2] It's time to notify for 515
Sep 22 23:22:24.229 INFO Completion from [2] id:515 status:true
Sep 22 23:22:24.229 INFO [516/752] Repair commands completed
Sep 22 23:22:24.229 INFO Pop front: ReconcileIO { id: ReconciliationId(516), op: ExtentFlush { repair_id: ReconciliationId(516), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.229 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.229 INFO [0] received reconcile message
Sep 22 23:22:24.229 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(516), op: ExtentFlush { repair_id: ReconciliationId(516), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.229 INFO [0] client ExtentFlush { repair_id: ReconciliationId(516), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.229 INFO [1] received reconcile message
Sep 22 23:22:24.229 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(516), op: ExtentFlush { repair_id: ReconciliationId(516), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.229 INFO [1] client ExtentFlush { repair_id: ReconciliationId(516), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.229 INFO [2] received reconcile message
Sep 22 23:22:24.229 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(516), op: ExtentFlush { repair_id: ReconciliationId(516), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.229 INFO [2] client ExtentFlush { repair_id: ReconciliationId(516), extent_id: 13, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.229 DEBG 516 Flush extent 13 with f:2 g:2
Sep 22 23:22:24.229 DEBG Flush just extent 13 with f:2 and g:2
Sep 22 23:22:24.229 DEBG [1] It's time to notify for 516
Sep 22 23:22:24.229 INFO Completion from [1] id:516 status:true
Sep 22 23:22:24.229 INFO [517/752] Repair commands completed
Sep 22 23:22:24.229 INFO Pop front: ReconcileIO { id: ReconciliationId(517), op: ExtentClose { repair_id: ReconciliationId(517), extent_id: 13 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.229 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.229 INFO [0] received reconcile message
Sep 22 23:22:24.229 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(517), op: ExtentClose { repair_id: ReconciliationId(517), extent_id: 13 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.229 INFO [0] client ExtentClose { repair_id: ReconciliationId(517), extent_id: 13 }
Sep 22 23:22:24.230 INFO [1] received reconcile message
Sep 22 23:22:24.230 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(517), op: ExtentClose { repair_id: ReconciliationId(517), extent_id: 13 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.230 INFO [1] client ExtentClose { repair_id: ReconciliationId(517), extent_id: 13 }
Sep 22 23:22:24.230 INFO [2] received reconcile message
Sep 22 23:22:24.230 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(517), op: ExtentClose { repair_id: ReconciliationId(517), extent_id: 13 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.230 INFO [2] client ExtentClose { repair_id: ReconciliationId(517), extent_id: 13 }
Sep 22 23:22:24.230 DEBG 517 Close extent 13
Sep 22 23:22:24.230 DEBG 517 Close extent 13
Sep 22 23:22:24.230 DEBG 517 Close extent 13
Sep 22 23:22:24.231 DEBG [2] It's time to notify for 517
Sep 22 23:22:24.231 INFO Completion from [2] id:517 status:true
Sep 22 23:22:24.231 INFO [518/752] Repair commands completed
Sep 22 23:22:24.231 INFO Pop front: ReconcileIO { id: ReconciliationId(518), op: ExtentRepair { repair_id: ReconciliationId(518), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.231 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.231 INFO [0] received reconcile message
Sep 22 23:22:24.231 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(518), op: ExtentRepair { repair_id: ReconciliationId(518), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.231 INFO [0] client ExtentRepair { repair_id: ReconciliationId(518), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.231 INFO [0] Sending repair request ReconciliationId(518)
Sep 22 23:22:24.231 INFO [1] received reconcile message
Sep 22 23:22:24.231 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(518), op: ExtentRepair { repair_id: ReconciliationId(518), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.231 INFO [1] client ExtentRepair { repair_id: ReconciliationId(518), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.231 INFO [1] No action required ReconciliationId(518)
Sep 22 23:22:24.231 INFO [2] received reconcile message
Sep 22 23:22:24.231 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(518), op: ExtentRepair { repair_id: ReconciliationId(518), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.231 INFO [2] client ExtentRepair { repair_id: ReconciliationId(518), extent_id: 13, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.231 INFO [2] No action required ReconciliationId(518)
Sep 22 23:22:24.231 DEBG 518 Repair extent 13 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.231 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/00D.copy"
Sep 22 23:22:24.294 INFO accepted connection, remote_addr: 127.0.0.1:41702, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.294 TRCE incoming request, uri: /extent/13/files, method: GET, req_id: 66e50e4a-25af-4e93-9e9c-693963a83856, remote_addr: 127.0.0.1:41702, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.294 INFO request completed, latency_us: 212, response_code: 200, uri: /extent/13/files, method: GET, req_id: 66e50e4a-25af-4e93-9e9c-693963a83856, remote_addr: 127.0.0.1:41702, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.294 INFO eid:13 Found repair files: ["00D", "00D.db"]
Sep 22 23:22:24.295 TRCE incoming request, uri: /newextent/13/data, method: GET, req_id: 69b94a42-ffb2-4a3c-a4da-842d66aa8f9b, remote_addr: 127.0.0.1:41702, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.295 INFO request completed, latency_us: 270, response_code: 200, uri: /newextent/13/data, method: GET, req_id: 69b94a42-ffb2-4a3c-a4da-842d66aa8f9b, remote_addr: 127.0.0.1:41702, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.300 TRCE incoming request, uri: /newextent/13/db, method: GET, req_id: 733f1b24-65b4-4712-a39d-e45b8a863647, remote_addr: 127.0.0.1:41702, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.300 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/13/db, method: GET, req_id: 733f1b24-65b4-4712-a39d-e45b8a863647, remote_addr: 127.0.0.1:41702, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.301 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/00D.copy" to "/tmp/downstairs-zrMnlo6G/00/000/00D.replace"
Sep 22 23:22:24.301 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.302 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/00D.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.302 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00D"
Sep 22 23:22:24.302 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00D.db"
Sep 22 23:22:24.302 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.302 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/00D.replace" to "/tmp/downstairs-zrMnlo6G/00/000/00D.completed"
Sep 22 23:22:24.302 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.302 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.302 DEBG [0] It's time to notify for 518
Sep 22 23:22:24.303 INFO Completion from [0] id:518 status:true
Sep 22 23:22:24.303 INFO [519/752] Repair commands completed
Sep 22 23:22:24.303 INFO Pop front: ReconcileIO { id: ReconciliationId(519), op: ExtentReopen { repair_id: ReconciliationId(519), extent_id: 13 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.303 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.303 INFO [0] received reconcile message
Sep 22 23:22:24.303 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(519), op: ExtentReopen { repair_id: ReconciliationId(519), extent_id: 13 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.303 INFO [0] client ExtentReopen { repair_id: ReconciliationId(519), extent_id: 13 }
Sep 22 23:22:24.303 INFO [1] received reconcile message
Sep 22 23:22:24.303 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(519), op: ExtentReopen { repair_id: ReconciliationId(519), extent_id: 13 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.303 INFO [1] client ExtentReopen { repair_id: ReconciliationId(519), extent_id: 13 }
Sep 22 23:22:24.303 INFO [2] received reconcile message
Sep 22 23:22:24.303 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(519), op: ExtentReopen { repair_id: ReconciliationId(519), extent_id: 13 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.303 INFO [2] client ExtentReopen { repair_id: ReconciliationId(519), extent_id: 13 }
Sep 22 23:22:24.303 DEBG 519 Reopen extent 13
Sep 22 23:22:24.304 DEBG 519 Reopen extent 13
Sep 22 23:22:24.304 DEBG 519 Reopen extent 13
Sep 22 23:22:24.305 DEBG [2] It's time to notify for 519
Sep 22 23:22:24.305 INFO Completion from [2] id:519 status:true
Sep 22 23:22:24.305 INFO [520/752] Repair commands completed
Sep 22 23:22:24.305 INFO Pop front: ReconcileIO { id: ReconciliationId(520), op: ExtentFlush { repair_id: ReconciliationId(520), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.305 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.305 INFO [0] received reconcile message
Sep 22 23:22:24.305 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(520), op: ExtentFlush { repair_id: ReconciliationId(520), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.305 INFO [0] client ExtentFlush { repair_id: ReconciliationId(520), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.305 INFO [1] received reconcile message
Sep 22 23:22:24.305 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(520), op: ExtentFlush { repair_id: ReconciliationId(520), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.305 INFO [1] client ExtentFlush { repair_id: ReconciliationId(520), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.305 INFO [2] received reconcile message
Sep 22 23:22:24.305 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(520), op: ExtentFlush { repair_id: ReconciliationId(520), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.305 INFO [2] client ExtentFlush { repair_id: ReconciliationId(520), extent_id: 29, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.305 DEBG 520 Flush extent 29 with f:2 g:2
Sep 22 23:22:24.305 DEBG Flush just extent 29 with f:2 and g:2
Sep 22 23:22:24.305 DEBG [1] It's time to notify for 520
Sep 22 23:22:24.305 INFO Completion from [1] id:520 status:true
Sep 22 23:22:24.306 INFO [521/752] Repair commands completed
Sep 22 23:22:24.306 INFO Pop front: ReconcileIO { id: ReconciliationId(521), op: ExtentClose { repair_id: ReconciliationId(521), extent_id: 29 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.306 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.306 INFO [0] received reconcile message
Sep 22 23:22:24.306 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(521), op: ExtentClose { repair_id: ReconciliationId(521), extent_id: 29 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.306 INFO [0] client ExtentClose { repair_id: ReconciliationId(521), extent_id: 29 }
Sep 22 23:22:24.306 INFO [1] received reconcile message
Sep 22 23:22:24.306 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(521), op: ExtentClose { repair_id: ReconciliationId(521), extent_id: 29 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.306 INFO [1] client ExtentClose { repair_id: ReconciliationId(521), extent_id: 29 }
Sep 22 23:22:24.306 INFO [2] received reconcile message
Sep 22 23:22:24.306 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(521), op: ExtentClose { repair_id: ReconciliationId(521), extent_id: 29 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.306 INFO [2] client ExtentClose { repair_id: ReconciliationId(521), extent_id: 29 }
Sep 22 23:22:24.306 DEBG 521 Close extent 29
Sep 22 23:22:24.306 DEBG 521 Close extent 29
Sep 22 23:22:24.306 DEBG 521 Close extent 29
Sep 22 23:22:24.307 DEBG [2] It's time to notify for 521
Sep 22 23:22:24.307 INFO Completion from [2] id:521 status:true
Sep 22 23:22:24.307 INFO [522/752] Repair commands completed
Sep 22 23:22:24.307 INFO Pop front: ReconcileIO { id: ReconciliationId(522), op: ExtentRepair { repair_id: ReconciliationId(522), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.307 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.307 INFO [0] received reconcile message
Sep 22 23:22:24.307 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(522), op: ExtentRepair { repair_id: ReconciliationId(522), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.307 INFO [0] client ExtentRepair { repair_id: ReconciliationId(522), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.307 INFO [0] Sending repair request ReconciliationId(522)
Sep 22 23:22:24.307 INFO [1] received reconcile message
Sep 22 23:22:24.307 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(522), op: ExtentRepair { repair_id: ReconciliationId(522), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.307 INFO [1] client ExtentRepair { repair_id: ReconciliationId(522), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.307 INFO [1] No action required ReconciliationId(522)
Sep 22 23:22:24.307 INFO [2] received reconcile message
Sep 22 23:22:24.307 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(522), op: ExtentRepair { repair_id: ReconciliationId(522), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.307 INFO [2] client ExtentRepair { repair_id: ReconciliationId(522), extent_id: 29, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.307 INFO [2] No action required ReconciliationId(522)
Sep 22 23:22:24.307 DEBG 522 Repair extent 29 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.307 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/01D.copy"
Sep 22 23:22:24.314 ERRO [2] job id 1085 saw error GenericError("test error")
Sep 22 23:22:24.314 ERRO [2] job id 1085 saw error GenericError("test error")
Sep 22 23:22:24.320 DEBG up_ds_listen was notified
Sep 22 23:22:24.320 DEBG up_ds_listen process 1088
Sep 22 23:22:24.320 DEBG [A] ack job 1088:89, : downstairs
Sep 22 23:22:24.320 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:24.327 DEBG IO Flush 1090 has deps [JobId(1089), JobId(1088)]
Sep 22 23:22:24.329 INFO [lossy] sleeping 1 second
Sep 22 23:22:24.370 INFO accepted connection, remote_addr: 127.0.0.1:58044, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.370 TRCE incoming request, uri: /extent/29/files, method: GET, req_id: 5b24ac0d-6762-4c06-bdbc-9c46d0044fe1, remote_addr: 127.0.0.1:58044, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.370 INFO request completed, latency_us: 279, response_code: 200, uri: /extent/29/files, method: GET, req_id: 5b24ac0d-6762-4c06-bdbc-9c46d0044fe1, remote_addr: 127.0.0.1:58044, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.371 INFO eid:29 Found repair files: ["01D", "01D.db"]
Sep 22 23:22:24.371 TRCE incoming request, uri: /newextent/29/data, method: GET, req_id: 6ea349d4-e9b2-406f-ae0b-65c5cfde5944, remote_addr: 127.0.0.1:58044, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.371 INFO request completed, latency_us: 376, response_code: 200, uri: /newextent/29/data, method: GET, req_id: 6ea349d4-e9b2-406f-ae0b-65c5cfde5944, remote_addr: 127.0.0.1:58044, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.377 TRCE incoming request, uri: /newextent/29/db, method: GET, req_id: 9833405e-3a0c-4085-8337-831aa02485cf, remote_addr: 127.0.0.1:58044, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.377 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/29/db, method: GET, req_id: 9833405e-3a0c-4085-8337-831aa02485cf, remote_addr: 127.0.0.1:58044, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.378 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/01D.copy" to "/tmp/downstairs-zrMnlo6G/00/000/01D.replace"
Sep 22 23:22:24.378 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.380 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/01D.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.380 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01D"
Sep 22 23:22:24.380 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01D.db"
Sep 22 23:22:24.380 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.380 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/01D.replace" to "/tmp/downstairs-zrMnlo6G/00/000/01D.completed"
Sep 22 23:22:24.380 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.380 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.380 DEBG [0] It's time to notify for 522
Sep 22 23:22:24.380 INFO Completion from [0] id:522 status:true
Sep 22 23:22:24.381 INFO [523/752] Repair commands completed
Sep 22 23:22:24.381 INFO Pop front: ReconcileIO { id: ReconciliationId(523), op: ExtentReopen { repair_id: ReconciliationId(523), extent_id: 29 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.381 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.381 INFO [0] received reconcile message
Sep 22 23:22:24.381 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(523), op: ExtentReopen { repair_id: ReconciliationId(523), extent_id: 29 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.381 INFO [0] client ExtentReopen { repair_id: ReconciliationId(523), extent_id: 29 }
Sep 22 23:22:24.381 INFO [1] received reconcile message
Sep 22 23:22:24.381 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(523), op: ExtentReopen { repair_id: ReconciliationId(523), extent_id: 29 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.381 INFO [1] client ExtentReopen { repair_id: ReconciliationId(523), extent_id: 29 }
Sep 22 23:22:24.381 INFO [2] received reconcile message
Sep 22 23:22:24.381 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(523), op: ExtentReopen { repair_id: ReconciliationId(523), extent_id: 29 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.381 INFO [2] client ExtentReopen { repair_id: ReconciliationId(523), extent_id: 29 }
Sep 22 23:22:24.381 DEBG 523 Reopen extent 29
Sep 22 23:22:24.382 DEBG 523 Reopen extent 29
Sep 22 23:22:24.382 DEBG 523 Reopen extent 29
Sep 22 23:22:24.383 DEBG [2] It's time to notify for 523
Sep 22 23:22:24.383 INFO Completion from [2] id:523 status:true
Sep 22 23:22:24.383 INFO [524/752] Repair commands completed
Sep 22 23:22:24.383 INFO Pop front: ReconcileIO { id: ReconciliationId(524), op: ExtentFlush { repair_id: ReconciliationId(524), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.383 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.383 INFO [0] received reconcile message
Sep 22 23:22:24.383 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(524), op: ExtentFlush { repair_id: ReconciliationId(524), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.383 INFO [0] client ExtentFlush { repair_id: ReconciliationId(524), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.383 INFO [1] received reconcile message
Sep 22 23:22:24.383 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(524), op: ExtentFlush { repair_id: ReconciliationId(524), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.383 INFO [1] client ExtentFlush { repair_id: ReconciliationId(524), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.383 INFO [2] received reconcile message
Sep 22 23:22:24.383 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(524), op: ExtentFlush { repair_id: ReconciliationId(524), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.383 INFO [2] client ExtentFlush { repair_id: ReconciliationId(524), extent_id: 27, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.383 DEBG 524 Flush extent 27 with f:2 g:2
Sep 22 23:22:24.383 DEBG Flush just extent 27 with f:2 and g:2
Sep 22 23:22:24.384 DEBG [1] It's time to notify for 524
Sep 22 23:22:24.384 INFO Completion from [1] id:524 status:true
Sep 22 23:22:24.384 INFO [525/752] Repair commands completed
Sep 22 23:22:24.384 INFO Pop front: ReconcileIO { id: ReconciliationId(525), op: ExtentClose { repair_id: ReconciliationId(525), extent_id: 27 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.384 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.384 INFO [0] received reconcile message
Sep 22 23:22:24.384 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(525), op: ExtentClose { repair_id: ReconciliationId(525), extent_id: 27 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.384 INFO [0] client ExtentClose { repair_id: ReconciliationId(525), extent_id: 27 }
Sep 22 23:22:24.384 INFO [1] received reconcile message
Sep 22 23:22:24.384 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(525), op: ExtentClose { repair_id: ReconciliationId(525), extent_id: 27 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.384 INFO [1] client ExtentClose { repair_id: ReconciliationId(525), extent_id: 27 }
Sep 22 23:22:24.384 INFO [2] received reconcile message
Sep 22 23:22:24.384 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(525), op: ExtentClose { repair_id: ReconciliationId(525), extent_id: 27 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.384 INFO [2] client ExtentClose { repair_id: ReconciliationId(525), extent_id: 27 }
Sep 22 23:22:24.384 DEBG 525 Close extent 27
Sep 22 23:22:24.384 DEBG 525 Close extent 27
Sep 22 23:22:24.385 DEBG 525 Close extent 27
Sep 22 23:22:24.385 DEBG [2] It's time to notify for 525
Sep 22 23:22:24.385 INFO Completion from [2] id:525 status:true
Sep 22 23:22:24.385 INFO [526/752] Repair commands completed
Sep 22 23:22:24.385 INFO Pop front: ReconcileIO { id: ReconciliationId(526), op: ExtentRepair { repair_id: ReconciliationId(526), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.385 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.385 INFO [0] received reconcile message
Sep 22 23:22:24.385 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(526), op: ExtentRepair { repair_id: ReconciliationId(526), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.385 INFO [0] client ExtentRepair { repair_id: ReconciliationId(526), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.385 INFO [0] Sending repair request ReconciliationId(526)
Sep 22 23:22:24.385 INFO [1] received reconcile message
Sep 22 23:22:24.385 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(526), op: ExtentRepair { repair_id: ReconciliationId(526), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.385 INFO [1] client ExtentRepair { repair_id: ReconciliationId(526), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.385 INFO [1] No action required ReconciliationId(526)
Sep 22 23:22:24.385 INFO [2] received reconcile message
Sep 22 23:22:24.385 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(526), op: ExtentRepair { repair_id: ReconciliationId(526), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.385 INFO [2] client ExtentRepair { repair_id: ReconciliationId(526), extent_id: 27, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.385 INFO [2] No action required ReconciliationId(526)
Sep 22 23:22:24.386 DEBG 526 Repair extent 27 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.386 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/01B.copy"
Sep 22 23:22:24.451 INFO accepted connection, remote_addr: 127.0.0.1:40549, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.451 TRCE incoming request, uri: /extent/27/files, method: GET, req_id: 0bbb56be-851b-4d16-83b4-6920e2d4d8e9, remote_addr: 127.0.0.1:40549, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.451 INFO request completed, latency_us: 280, response_code: 200, uri: /extent/27/files, method: GET, req_id: 0bbb56be-851b-4d16-83b4-6920e2d4d8e9, remote_addr: 127.0.0.1:40549, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.451 INFO eid:27 Found repair files: ["01B", "01B.db"]
Sep 22 23:22:24.452 TRCE incoming request, uri: /newextent/27/data, method: GET, req_id: 4675541f-33ad-4c07-b124-2ea7596493ba, remote_addr: 127.0.0.1:40549, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.452 INFO request completed, latency_us: 371, response_code: 200, uri: /newextent/27/data, method: GET, req_id: 4675541f-33ad-4c07-b124-2ea7596493ba, remote_addr: 127.0.0.1:40549, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.457 TRCE incoming request, uri: /newextent/27/db, method: GET, req_id: dd697937-a181-4f85-93cf-a1ae2707c48f, remote_addr: 127.0.0.1:40549, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.458 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/27/db, method: GET, req_id: dd697937-a181-4f85-93cf-a1ae2707c48f, remote_addr: 127.0.0.1:40549, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.459 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/01B.copy" to "/tmp/downstairs-zrMnlo6G/00/000/01B.replace"
Sep 22 23:22:24.459 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.460 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/01B.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.460 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01B"
Sep 22 23:22:24.460 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/01B.db"
Sep 22 23:22:24.460 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.460 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/01B.replace" to "/tmp/downstairs-zrMnlo6G/00/000/01B.completed"
Sep 22 23:22:24.460 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.460 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.461 DEBG [0] It's time to notify for 526
Sep 22 23:22:24.461 INFO Completion from [0] id:526 status:true
Sep 22 23:22:24.461 INFO [527/752] Repair commands completed
Sep 22 23:22:24.461 INFO Pop front: ReconcileIO { id: ReconciliationId(527), op: ExtentReopen { repair_id: ReconciliationId(527), extent_id: 27 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.461 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.461 INFO [0] received reconcile message
Sep 22 23:22:24.461 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(527), op: ExtentReopen { repair_id: ReconciliationId(527), extent_id: 27 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.461 INFO [0] client ExtentReopen { repair_id: ReconciliationId(527), extent_id: 27 }
Sep 22 23:22:24.461 INFO [1] received reconcile message
Sep 22 23:22:24.461 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(527), op: ExtentReopen { repair_id: ReconciliationId(527), extent_id: 27 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.461 INFO [1] client ExtentReopen { repair_id: ReconciliationId(527), extent_id: 27 }
Sep 22 23:22:24.461 INFO [2] received reconcile message
Sep 22 23:22:24.461 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(527), op: ExtentReopen { repair_id: ReconciliationId(527), extent_id: 27 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.461 INFO [2] client ExtentReopen { repair_id: ReconciliationId(527), extent_id: 27 }
Sep 22 23:22:24.461 DEBG 527 Reopen extent 27
Sep 22 23:22:24.462 DEBG 527 Reopen extent 27
Sep 22 23:22:24.463 DEBG 527 Reopen extent 27
Sep 22 23:22:24.463 DEBG [2] It's time to notify for 527
Sep 22 23:22:24.463 INFO Completion from [2] id:527 status:true
Sep 22 23:22:24.463 INFO [528/752] Repair commands completed
Sep 22 23:22:24.463 INFO Pop front: ReconcileIO { id: ReconciliationId(528), op: ExtentFlush { repair_id: ReconciliationId(528), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.463 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.463 INFO [0] received reconcile message
Sep 22 23:22:24.463 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(528), op: ExtentFlush { repair_id: ReconciliationId(528), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.463 INFO [0] client ExtentFlush { repair_id: ReconciliationId(528), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.464 INFO [1] received reconcile message
Sep 22 23:22:24.464 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(528), op: ExtentFlush { repair_id: ReconciliationId(528), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.464 INFO [1] client ExtentFlush { repair_id: ReconciliationId(528), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.464 INFO [2] received reconcile message
Sep 22 23:22:24.464 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(528), op: ExtentFlush { repair_id: ReconciliationId(528), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.464 INFO [2] client ExtentFlush { repair_id: ReconciliationId(528), extent_id: 187, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.464 DEBG 528 Flush extent 187 with f:2 g:2
Sep 22 23:22:24.464 DEBG Flush just extent 187 with f:2 and g:2
Sep 22 23:22:24.464 DEBG [1] It's time to notify for 528
Sep 22 23:22:24.464 INFO Completion from [1] id:528 status:true
Sep 22 23:22:24.464 INFO [529/752] Repair commands completed
Sep 22 23:22:24.464 INFO Pop front: ReconcileIO { id: ReconciliationId(529), op: ExtentClose { repair_id: ReconciliationId(529), extent_id: 187 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.464 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.464 INFO [0] received reconcile message
Sep 22 23:22:24.464 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(529), op: ExtentClose { repair_id: ReconciliationId(529), extent_id: 187 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.464 INFO [0] client ExtentClose { repair_id: ReconciliationId(529), extent_id: 187 }
Sep 22 23:22:24.464 INFO [1] received reconcile message
Sep 22 23:22:24.464 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(529), op: ExtentClose { repair_id: ReconciliationId(529), extent_id: 187 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.464 INFO [1] client ExtentClose { repair_id: ReconciliationId(529), extent_id: 187 }
Sep 22 23:22:24.464 INFO [2] received reconcile message
Sep 22 23:22:24.464 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(529), op: ExtentClose { repair_id: ReconciliationId(529), extent_id: 187 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.464 INFO [2] client ExtentClose { repair_id: ReconciliationId(529), extent_id: 187 }
Sep 22 23:22:24.464 DEBG 529 Close extent 187
Sep 22 23:22:24.465 DEBG 529 Close extent 187
Sep 22 23:22:24.465 DEBG 529 Close extent 187
Sep 22 23:22:24.465 DEBG [2] It's time to notify for 529
Sep 22 23:22:24.465 INFO Completion from [2] id:529 status:true
Sep 22 23:22:24.465 INFO [530/752] Repair commands completed
Sep 22 23:22:24.465 INFO Pop front: ReconcileIO { id: ReconciliationId(530), op: ExtentRepair { repair_id: ReconciliationId(530), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.465 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.465 INFO [0] received reconcile message
Sep 22 23:22:24.465 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(530), op: ExtentRepair { repair_id: ReconciliationId(530), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.465 INFO [0] client ExtentRepair { repair_id: ReconciliationId(530), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.466 INFO [0] Sending repair request ReconciliationId(530)
Sep 22 23:22:24.466 INFO [1] received reconcile message
Sep 22 23:22:24.466 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(530), op: ExtentRepair { repair_id: ReconciliationId(530), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.466 INFO [1] client ExtentRepair { repair_id: ReconciliationId(530), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.466 INFO [1] No action required ReconciliationId(530)
Sep 22 23:22:24.466 INFO [2] received reconcile message
Sep 22 23:22:24.466 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(530), op: ExtentRepair { repair_id: ReconciliationId(530), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.466 INFO [2] client ExtentRepair { repair_id: ReconciliationId(530), extent_id: 187, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.466 INFO [2] No action required ReconciliationId(530)
Sep 22 23:22:24.466 DEBG 530 Repair extent 187 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.466 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0BB.copy"
Sep 22 23:22:24.530 INFO accepted connection, remote_addr: 127.0.0.1:47833, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.530 TRCE incoming request, uri: /extent/187/files, method: GET, req_id: 6a5beff3-cbf4-4056-a311-fc7eb812ddb5, remote_addr: 127.0.0.1:47833, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.531 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/187/files, method: GET, req_id: 6a5beff3-cbf4-4056-a311-fc7eb812ddb5, remote_addr: 127.0.0.1:47833, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.531 INFO eid:187 Found repair files: ["0BB", "0BB.db"]
Sep 22 23:22:24.531 TRCE incoming request, uri: /newextent/187/data, method: GET, req_id: 654d7456-cdad-459c-8377-e3ad0cd45a8e, remote_addr: 127.0.0.1:47833, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.531 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/187/data, method: GET, req_id: 654d7456-cdad-459c-8377-e3ad0cd45a8e, remote_addr: 127.0.0.1:47833, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.536 TRCE incoming request, uri: /newextent/187/db, method: GET, req_id: d0be47de-4df2-4e8b-a627-5f840b365d52, remote_addr: 127.0.0.1:47833, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.537 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/187/db, method: GET, req_id: d0be47de-4df2-4e8b-a627-5f840b365d52, remote_addr: 127.0.0.1:47833, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.538 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0BB.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0BB.replace"
Sep 22 23:22:24.538 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.538 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0BB.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.539 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0BB"
Sep 22 23:22:24.539 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0BB.db"
Sep 22 23:22:24.539 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.539 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0BB.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0BB.completed"
Sep 22 23:22:24.539 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.539 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.539 DEBG [0] It's time to notify for 530
Sep 22 23:22:24.539 INFO Completion from [0] id:530 status:true
Sep 22 23:22:24.539 INFO [531/752] Repair commands completed
Sep 22 23:22:24.539 INFO Pop front: ReconcileIO { id: ReconciliationId(531), op: ExtentReopen { repair_id: ReconciliationId(531), extent_id: 187 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.539 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.539 INFO [0] received reconcile message
Sep 22 23:22:24.539 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(531), op: ExtentReopen { repair_id: ReconciliationId(531), extent_id: 187 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.539 INFO [0] client ExtentReopen { repair_id: ReconciliationId(531), extent_id: 187 }
Sep 22 23:22:24.539 INFO [1] received reconcile message
Sep 22 23:22:24.539 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(531), op: ExtentReopen { repair_id: ReconciliationId(531), extent_id: 187 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.539 INFO [1] client ExtentReopen { repair_id: ReconciliationId(531), extent_id: 187 }
Sep 22 23:22:24.539 INFO [2] received reconcile message
Sep 22 23:22:24.540 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(531), op: ExtentReopen { repair_id: ReconciliationId(531), extent_id: 187 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.540 INFO [2] client ExtentReopen { repair_id: ReconciliationId(531), extent_id: 187 }
Sep 22 23:22:24.540 DEBG 531 Reopen extent 187
Sep 22 23:22:24.540 DEBG 531 Reopen extent 187
Sep 22 23:22:24.541 DEBG 531 Reopen extent 187
Sep 22 23:22:24.541 DEBG [2] It's time to notify for 531
Sep 22 23:22:24.541 INFO Completion from [2] id:531 status:true
Sep 22 23:22:24.541 INFO [532/752] Repair commands completed
Sep 22 23:22:24.541 INFO Pop front: ReconcileIO { id: ReconciliationId(532), op: ExtentFlush { repair_id: ReconciliationId(532), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.541 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.541 INFO [0] received reconcile message
Sep 22 23:22:24.541 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(532), op: ExtentFlush { repair_id: ReconciliationId(532), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.541 INFO [0] client ExtentFlush { repair_id: ReconciliationId(532), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.541 INFO [1] received reconcile message
Sep 22 23:22:24.542 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(532), op: ExtentFlush { repair_id: ReconciliationId(532), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.542 INFO [1] client ExtentFlush { repair_id: ReconciliationId(532), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.542 INFO [2] received reconcile message
Sep 22 23:22:24.542 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(532), op: ExtentFlush { repair_id: ReconciliationId(532), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.542 INFO [2] client ExtentFlush { repair_id: ReconciliationId(532), extent_id: 10, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.542 DEBG 532 Flush extent 10 with f:2 g:2
Sep 22 23:22:24.542 DEBG Flush just extent 10 with f:2 and g:2
Sep 22 23:22:24.542 DEBG [1] It's time to notify for 532
Sep 22 23:22:24.542 INFO Completion from [1] id:532 status:true
Sep 22 23:22:24.542 INFO [533/752] Repair commands completed
Sep 22 23:22:24.542 INFO Pop front: ReconcileIO { id: ReconciliationId(533), op: ExtentClose { repair_id: ReconciliationId(533), extent_id: 10 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.542 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.542 INFO [0] received reconcile message
Sep 22 23:22:24.542 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(533), op: ExtentClose { repair_id: ReconciliationId(533), extent_id: 10 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.542 INFO [0] client ExtentClose { repair_id: ReconciliationId(533), extent_id: 10 }
Sep 22 23:22:24.542 INFO [1] received reconcile message
Sep 22 23:22:24.542 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(533), op: ExtentClose { repair_id: ReconciliationId(533), extent_id: 10 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.542 INFO [1] client ExtentClose { repair_id: ReconciliationId(533), extent_id: 10 }
Sep 22 23:22:24.542 INFO [2] received reconcile message
Sep 22 23:22:24.542 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(533), op: ExtentClose { repair_id: ReconciliationId(533), extent_id: 10 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.542 INFO [2] client ExtentClose { repair_id: ReconciliationId(533), extent_id: 10 }
Sep 22 23:22:24.542 DEBG 533 Close extent 10
Sep 22 23:22:24.543 DEBG 533 Close extent 10
Sep 22 23:22:24.543 DEBG 533 Close extent 10
Sep 22 23:22:24.543 DEBG [2] It's time to notify for 533
Sep 22 23:22:24.543 INFO Completion from [2] id:533 status:true
Sep 22 23:22:24.543 INFO [534/752] Repair commands completed
Sep 22 23:22:24.543 INFO Pop front: ReconcileIO { id: ReconciliationId(534), op: ExtentRepair { repair_id: ReconciliationId(534), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.543 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.543 INFO [0] received reconcile message
Sep 22 23:22:24.543 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(534), op: ExtentRepair { repair_id: ReconciliationId(534), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.543 INFO [0] client ExtentRepair { repair_id: ReconciliationId(534), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.543 INFO [0] Sending repair request ReconciliationId(534)
Sep 22 23:22:24.544 INFO [1] received reconcile message
Sep 22 23:22:24.544 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(534), op: ExtentRepair { repair_id: ReconciliationId(534), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.544 INFO [1] client ExtentRepair { repair_id: ReconciliationId(534), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.544 INFO [1] No action required ReconciliationId(534)
Sep 22 23:22:24.544 INFO [2] received reconcile message
Sep 22 23:22:24.544 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(534), op: ExtentRepair { repair_id: ReconciliationId(534), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.544 INFO [2] client ExtentRepair { repair_id: ReconciliationId(534), extent_id: 10, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.544 INFO [2] No action required ReconciliationId(534)
Sep 22 23:22:24.544 DEBG 534 Repair extent 10 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.544 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/00A.copy"
Sep 22 23:22:24.609 INFO accepted connection, remote_addr: 127.0.0.1:36657, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.609 TRCE incoming request, uri: /extent/10/files, method: GET, req_id: 0138adee-1167-4148-8bca-ac2c5a796e4d, remote_addr: 127.0.0.1:36657, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.610 INFO request completed, latency_us: 204, response_code: 200, uri: /extent/10/files, method: GET, req_id: 0138adee-1167-4148-8bca-ac2c5a796e4d, remote_addr: 127.0.0.1:36657, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.610 INFO eid:10 Found repair files: ["00A", "00A.db"]
Sep 22 23:22:24.610 TRCE incoming request, uri: /newextent/10/data, method: GET, req_id: a193fb1d-9cb5-4ff9-9c71-1cc771a39bea, remote_addr: 127.0.0.1:36657, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.610 INFO request completed, latency_us: 310, response_code: 200, uri: /newextent/10/data, method: GET, req_id: a193fb1d-9cb5-4ff9-9c71-1cc771a39bea, remote_addr: 127.0.0.1:36657, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.615 TRCE incoming request, uri: /newextent/10/db, method: GET, req_id: d464a382-75d2-4a9c-9d2b-f257a9c17983, remote_addr: 127.0.0.1:36657, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.616 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/10/db, method: GET, req_id: d464a382-75d2-4a9c-9d2b-f257a9c17983, remote_addr: 127.0.0.1:36657, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.617 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/00A.copy" to "/tmp/downstairs-zrMnlo6G/00/000/00A.replace"
Sep 22 23:22:24.617 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.617 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/00A.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.618 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00A"
Sep 22 23:22:24.618 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00A.db"
Sep 22 23:22:24.618 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.618 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/00A.replace" to "/tmp/downstairs-zrMnlo6G/00/000/00A.completed"
Sep 22 23:22:24.618 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.618 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.618 DEBG [0] It's time to notify for 534
Sep 22 23:22:24.618 INFO Completion from [0] id:534 status:true
Sep 22 23:22:24.618 INFO [535/752] Repair commands completed
Sep 22 23:22:24.618 INFO Pop front: ReconcileIO { id: ReconciliationId(535), op: ExtentReopen { repair_id: ReconciliationId(535), extent_id: 10 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.618 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.618 INFO [0] received reconcile message
Sep 22 23:22:24.618 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(535), op: ExtentReopen { repair_id: ReconciliationId(535), extent_id: 10 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.618 INFO [0] client ExtentReopen { repair_id: ReconciliationId(535), extent_id: 10 }
Sep 22 23:22:24.618 INFO [1] received reconcile message
Sep 22 23:22:24.618 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(535), op: ExtentReopen { repair_id: ReconciliationId(535), extent_id: 10 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.618 INFO [1] client ExtentReopen { repair_id: ReconciliationId(535), extent_id: 10 }
Sep 22 23:22:24.619 INFO [2] received reconcile message
Sep 22 23:22:24.619 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(535), op: ExtentReopen { repair_id: ReconciliationId(535), extent_id: 10 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.619 INFO [2] client ExtentReopen { repair_id: ReconciliationId(535), extent_id: 10 }
Sep 22 23:22:24.619 DEBG 535 Reopen extent 10
Sep 22 23:22:24.619 DEBG 535 Reopen extent 10
Sep 22 23:22:24.620 DEBG 535 Reopen extent 10
Sep 22 23:22:24.620 DEBG [2] It's time to notify for 535
Sep 22 23:22:24.620 INFO Completion from [2] id:535 status:true
Sep 22 23:22:24.621 INFO [536/752] Repair commands completed
Sep 22 23:22:24.621 INFO Pop front: ReconcileIO { id: ReconciliationId(536), op: ExtentFlush { repair_id: ReconciliationId(536), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.621 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.621 INFO [0] received reconcile message
Sep 22 23:22:24.621 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(536), op: ExtentFlush { repair_id: ReconciliationId(536), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.621 INFO [0] client ExtentFlush { repair_id: ReconciliationId(536), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.621 INFO [1] received reconcile message
Sep 22 23:22:24.621 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(536), op: ExtentFlush { repair_id: ReconciliationId(536), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.621 INFO [1] client ExtentFlush { repair_id: ReconciliationId(536), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.621 INFO [2] received reconcile message
Sep 22 23:22:24.621 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(536), op: ExtentFlush { repair_id: ReconciliationId(536), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.621 INFO [2] client ExtentFlush { repair_id: ReconciliationId(536), extent_id: 102, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.621 DEBG 536 Flush extent 102 with f:2 g:2
Sep 22 23:22:24.621 DEBG Flush just extent 102 with f:2 and g:2
Sep 22 23:22:24.621 DEBG [1] It's time to notify for 536
Sep 22 23:22:24.621 INFO Completion from [1] id:536 status:true
Sep 22 23:22:24.621 INFO [537/752] Repair commands completed
Sep 22 23:22:24.621 INFO Pop front: ReconcileIO { id: ReconciliationId(537), op: ExtentClose { repair_id: ReconciliationId(537), extent_id: 102 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.621 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.621 INFO [0] received reconcile message
Sep 22 23:22:24.621 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(537), op: ExtentClose { repair_id: ReconciliationId(537), extent_id: 102 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.621 INFO [0] client ExtentClose { repair_id: ReconciliationId(537), extent_id: 102 }
Sep 22 23:22:24.621 INFO [1] received reconcile message
Sep 22 23:22:24.621 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(537), op: ExtentClose { repair_id: ReconciliationId(537), extent_id: 102 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.621 INFO [1] client ExtentClose { repair_id: ReconciliationId(537), extent_id: 102 }
Sep 22 23:22:24.621 INFO [2] received reconcile message
Sep 22 23:22:24.621 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(537), op: ExtentClose { repair_id: ReconciliationId(537), extent_id: 102 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.621 INFO [2] client ExtentClose { repair_id: ReconciliationId(537), extent_id: 102 }
Sep 22 23:22:24.622 DEBG 537 Close extent 102
Sep 22 23:22:24.622 DEBG 537 Close extent 102
Sep 22 23:22:24.622 DEBG 537 Close extent 102
Sep 22 23:22:24.622 DEBG [2] It's time to notify for 537
Sep 22 23:22:24.623 INFO Completion from [2] id:537 status:true
Sep 22 23:22:24.623 INFO [538/752] Repair commands completed
Sep 22 23:22:24.623 INFO Pop front: ReconcileIO { id: ReconciliationId(538), op: ExtentRepair { repair_id: ReconciliationId(538), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.623 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.623 INFO [0] received reconcile message
Sep 22 23:22:24.623 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(538), op: ExtentRepair { repair_id: ReconciliationId(538), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.623 INFO [0] client ExtentRepair { repair_id: ReconciliationId(538), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.623 INFO [0] Sending repair request ReconciliationId(538)
Sep 22 23:22:24.623 INFO [1] received reconcile message
Sep 22 23:22:24.623 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(538), op: ExtentRepair { repair_id: ReconciliationId(538), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.623 INFO [1] client ExtentRepair { repair_id: ReconciliationId(538), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.623 INFO [1] No action required ReconciliationId(538)
Sep 22 23:22:24.623 INFO [2] received reconcile message
Sep 22 23:22:24.623 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(538), op: ExtentRepair { repair_id: ReconciliationId(538), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.623 INFO [2] client ExtentRepair { repair_id: ReconciliationId(538), extent_id: 102, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.623 INFO [2] No action required ReconciliationId(538)
Sep 22 23:22:24.623 DEBG 538 Repair extent 102 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.623 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/066.copy"
Sep 22 23:22:24.687 INFO accepted connection, remote_addr: 127.0.0.1:60465, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.687 TRCE incoming request, uri: /extent/102/files, method: GET, req_id: 652d8c9c-1f13-4868-8cae-847f70b5d42f, remote_addr: 127.0.0.1:60465, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.687 INFO request completed, latency_us: 202, response_code: 200, uri: /extent/102/files, method: GET, req_id: 652d8c9c-1f13-4868-8cae-847f70b5d42f, remote_addr: 127.0.0.1:60465, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.688 INFO eid:102 Found repair files: ["066", "066.db"]
Sep 22 23:22:24.688 TRCE incoming request, uri: /newextent/102/data, method: GET, req_id: 4318ae0c-e308-4667-b91e-eeeaaaef7ef5, remote_addr: 127.0.0.1:60465, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.688 INFO request completed, latency_us: 325, response_code: 200, uri: /newextent/102/data, method: GET, req_id: 4318ae0c-e308-4667-b91e-eeeaaaef7ef5, remote_addr: 127.0.0.1:60465, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.693 TRCE incoming request, uri: /newextent/102/db, method: GET, req_id: 7058aed6-2f4d-4842-a128-4e21b6f2e1fe, remote_addr: 127.0.0.1:60465, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.693 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/102/db, method: GET, req_id: 7058aed6-2f4d-4842-a128-4e21b6f2e1fe, remote_addr: 127.0.0.1:60465, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.694 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/066.copy" to "/tmp/downstairs-zrMnlo6G/00/000/066.replace"
Sep 22 23:22:24.694 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.695 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/066.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.695 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/066"
Sep 22 23:22:24.695 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/066.db"
Sep 22 23:22:24.695 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.695 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/066.replace" to "/tmp/downstairs-zrMnlo6G/00/000/066.completed"
Sep 22 23:22:24.696 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.696 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.696 DEBG [0] It's time to notify for 538
Sep 22 23:22:24.696 INFO Completion from [0] id:538 status:true
Sep 22 23:22:24.696 INFO [539/752] Repair commands completed
Sep 22 23:22:24.696 INFO Pop front: ReconcileIO { id: ReconciliationId(539), op: ExtentReopen { repair_id: ReconciliationId(539), extent_id: 102 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.696 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.696 INFO [0] received reconcile message
Sep 22 23:22:24.696 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(539), op: ExtentReopen { repair_id: ReconciliationId(539), extent_id: 102 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.696 INFO [0] client ExtentReopen { repair_id: ReconciliationId(539), extent_id: 102 }
Sep 22 23:22:24.696 INFO [1] received reconcile message
Sep 22 23:22:24.696 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(539), op: ExtentReopen { repair_id: ReconciliationId(539), extent_id: 102 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.696 INFO [1] client ExtentReopen { repair_id: ReconciliationId(539), extent_id: 102 }
Sep 22 23:22:24.696 INFO [2] received reconcile message
Sep 22 23:22:24.696 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(539), op: ExtentReopen { repair_id: ReconciliationId(539), extent_id: 102 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.696 INFO [2] client ExtentReopen { repair_id: ReconciliationId(539), extent_id: 102 }
Sep 22 23:22:24.696 DEBG 539 Reopen extent 102
Sep 22 23:22:24.697 DEBG 539 Reopen extent 102
Sep 22 23:22:24.698 DEBG 539 Reopen extent 102
Sep 22 23:22:24.698 DEBG [2] It's time to notify for 539
Sep 22 23:22:24.698 INFO Completion from [2] id:539 status:true
Sep 22 23:22:24.698 INFO [540/752] Repair commands completed
Sep 22 23:22:24.698 INFO Pop front: ReconcileIO { id: ReconciliationId(540), op: ExtentFlush { repair_id: ReconciliationId(540), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.698 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.698 INFO [0] received reconcile message
Sep 22 23:22:24.698 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(540), op: ExtentFlush { repair_id: ReconciliationId(540), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.698 INFO [0] client ExtentFlush { repair_id: ReconciliationId(540), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.698 INFO [1] received reconcile message
Sep 22 23:22:24.698 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(540), op: ExtentFlush { repair_id: ReconciliationId(540), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.698 INFO [1] client ExtentFlush { repair_id: ReconciliationId(540), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.698 INFO [2] received reconcile message
Sep 22 23:22:24.698 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(540), op: ExtentFlush { repair_id: ReconciliationId(540), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.698 INFO [2] client ExtentFlush { repair_id: ReconciliationId(540), extent_id: 85, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.699 DEBG 540 Flush extent 85 with f:2 g:2
Sep 22 23:22:24.699 DEBG Flush just extent 85 with f:2 and g:2
Sep 22 23:22:24.699 DEBG [1] It's time to notify for 540
Sep 22 23:22:24.699 INFO Completion from [1] id:540 status:true
Sep 22 23:22:24.699 INFO [541/752] Repair commands completed
Sep 22 23:22:24.699 INFO Pop front: ReconcileIO { id: ReconciliationId(541), op: ExtentClose { repair_id: ReconciliationId(541), extent_id: 85 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.699 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.699 INFO [0] received reconcile message
Sep 22 23:22:24.699 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(541), op: ExtentClose { repair_id: ReconciliationId(541), extent_id: 85 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.699 INFO [0] client ExtentClose { repair_id: ReconciliationId(541), extent_id: 85 }
Sep 22 23:22:24.699 INFO [1] received reconcile message
Sep 22 23:22:24.699 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(541), op: ExtentClose { repair_id: ReconciliationId(541), extent_id: 85 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.699 INFO [1] client ExtentClose { repair_id: ReconciliationId(541), extent_id: 85 }
Sep 22 23:22:24.699 INFO [2] received reconcile message
Sep 22 23:22:24.699 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(541), op: ExtentClose { repair_id: ReconciliationId(541), extent_id: 85 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.699 INFO [2] client ExtentClose { repair_id: ReconciliationId(541), extent_id: 85 }
Sep 22 23:22:24.699 DEBG 541 Close extent 85
Sep 22 23:22:24.699 DEBG 541 Close extent 85
Sep 22 23:22:24.700 DEBG 541 Close extent 85
Sep 22 23:22:24.700 DEBG [2] It's time to notify for 541
Sep 22 23:22:24.700 INFO Completion from [2] id:541 status:true
Sep 22 23:22:24.700 INFO [542/752] Repair commands completed
Sep 22 23:22:24.700 INFO Pop front: ReconcileIO { id: ReconciliationId(542), op: ExtentRepair { repair_id: ReconciliationId(542), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.700 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.700 INFO [0] received reconcile message
Sep 22 23:22:24.700 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(542), op: ExtentRepair { repair_id: ReconciliationId(542), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.700 INFO [0] client ExtentRepair { repair_id: ReconciliationId(542), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.700 INFO [0] Sending repair request ReconciliationId(542)
Sep 22 23:22:24.700 INFO [1] received reconcile message
Sep 22 23:22:24.700 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(542), op: ExtentRepair { repair_id: ReconciliationId(542), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.700 INFO [1] client ExtentRepair { repair_id: ReconciliationId(542), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.700 INFO [1] No action required ReconciliationId(542)
Sep 22 23:22:24.700 INFO [2] received reconcile message
Sep 22 23:22:24.701 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(542), op: ExtentRepair { repair_id: ReconciliationId(542), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.701 INFO [2] client ExtentRepair { repair_id: ReconciliationId(542), extent_id: 85, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.701 INFO [2] No action required ReconciliationId(542)
Sep 22 23:22:24.701 DEBG 542 Repair extent 85 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.701 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/055.copy"
Sep 22 23:22:24.711 INFO [lossy] skipping 1089
Sep 22 23:22:24.711 INFO [lossy] skipping 1090
Sep 22 23:22:24.716 DEBG Read      :1089 deps:[JobId(1088)] res:true
Sep 22 23:22:24.739 WARN returning error on flush!
Sep 22 23:22:24.739 DEBG Flush     :1090 extent_limit None deps:[JobId(1089), JobId(1088)] res:false f:34 g:1
Sep 22 23:22:24.739 INFO [lossy] skipping 1090
Sep 22 23:22:24.739 DEBG Flush     :1090 extent_limit None deps:[JobId(1089), JobId(1088)] res:true f:34 g:1
Sep 22 23:22:24.739 INFO [lossy] sleeping 1 second
Sep 22 23:22:24.766 INFO accepted connection, remote_addr: 127.0.0.1:60551, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.767 TRCE incoming request, uri: /extent/85/files, method: GET, req_id: 66ad53f4-4822-41c4-a1fd-daef5bd37365, remote_addr: 127.0.0.1:60551, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.767 INFO request completed, latency_us: 224, response_code: 200, uri: /extent/85/files, method: GET, req_id: 66ad53f4-4822-41c4-a1fd-daef5bd37365, remote_addr: 127.0.0.1:60551, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.767 INFO eid:85 Found repair files: ["055", "055.db"]
Sep 22 23:22:24.767 TRCE incoming request, uri: /newextent/85/data, method: GET, req_id: 867be027-70ed-4524-8e97-3317cb147004, remote_addr: 127.0.0.1:60551, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.768 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/85/data, method: GET, req_id: 867be027-70ed-4524-8e97-3317cb147004, remote_addr: 127.0.0.1:60551, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.772 TRCE incoming request, uri: /newextent/85/db, method: GET, req_id: 9f388412-8208-4b14-8a1f-dba266bfb627, remote_addr: 127.0.0.1:60551, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.773 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/85/db, method: GET, req_id: 9f388412-8208-4b14-8a1f-dba266bfb627, remote_addr: 127.0.0.1:60551, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.774 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/055.copy" to "/tmp/downstairs-zrMnlo6G/00/000/055.replace"
Sep 22 23:22:24.774 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.775 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/055.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.775 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/055"
Sep 22 23:22:24.775 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/055.db"
Sep 22 23:22:24.775 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.775 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/055.replace" to "/tmp/downstairs-zrMnlo6G/00/000/055.completed"
Sep 22 23:22:24.775 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.775 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.775 DEBG [0] It's time to notify for 542
Sep 22 23:22:24.776 INFO Completion from [0] id:542 status:true
Sep 22 23:22:24.776 INFO [543/752] Repair commands completed
Sep 22 23:22:24.776 INFO Pop front: ReconcileIO { id: ReconciliationId(543), op: ExtentReopen { repair_id: ReconciliationId(543), extent_id: 85 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.776 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.776 INFO [0] received reconcile message
Sep 22 23:22:24.776 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(543), op: ExtentReopen { repair_id: ReconciliationId(543), extent_id: 85 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.776 INFO [0] client ExtentReopen { repair_id: ReconciliationId(543), extent_id: 85 }
Sep 22 23:22:24.776 INFO [1] received reconcile message
Sep 22 23:22:24.776 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(543), op: ExtentReopen { repair_id: ReconciliationId(543), extent_id: 85 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.776 INFO [1] client ExtentReopen { repair_id: ReconciliationId(543), extent_id: 85 }
Sep 22 23:22:24.776 INFO [2] received reconcile message
Sep 22 23:22:24.776 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(543), op: ExtentReopen { repair_id: ReconciliationId(543), extent_id: 85 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.776 INFO [2] client ExtentReopen { repair_id: ReconciliationId(543), extent_id: 85 }
Sep 22 23:22:24.776 DEBG 543 Reopen extent 85
Sep 22 23:22:24.777 DEBG 543 Reopen extent 85
Sep 22 23:22:24.777 DEBG 543 Reopen extent 85
Sep 22 23:22:24.778 DEBG [2] It's time to notify for 543
Sep 22 23:22:24.778 INFO Completion from [2] id:543 status:true
Sep 22 23:22:24.778 INFO [544/752] Repair commands completed
Sep 22 23:22:24.778 INFO Pop front: ReconcileIO { id: ReconciliationId(544), op: ExtentFlush { repair_id: ReconciliationId(544), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.778 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.778 INFO [0] received reconcile message
Sep 22 23:22:24.778 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(544), op: ExtentFlush { repair_id: ReconciliationId(544), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.778 INFO [0] client ExtentFlush { repair_id: ReconciliationId(544), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.778 INFO [1] received reconcile message
Sep 22 23:22:24.778 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(544), op: ExtentFlush { repair_id: ReconciliationId(544), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.778 INFO [1] client ExtentFlush { repair_id: ReconciliationId(544), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.778 INFO [2] received reconcile message
Sep 22 23:22:24.778 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(544), op: ExtentFlush { repair_id: ReconciliationId(544), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.778 INFO [2] client ExtentFlush { repair_id: ReconciliationId(544), extent_id: 100, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.778 DEBG 544 Flush extent 100 with f:2 g:2
Sep 22 23:22:24.778 DEBG Flush just extent 100 with f:2 and g:2
Sep 22 23:22:24.778 DEBG [1] It's time to notify for 544
Sep 22 23:22:24.778 INFO Completion from [1] id:544 status:true
Sep 22 23:22:24.778 INFO [545/752] Repair commands completed
Sep 22 23:22:24.778 INFO Pop front: ReconcileIO { id: ReconciliationId(545), op: ExtentClose { repair_id: ReconciliationId(545), extent_id: 100 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.778 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.778 INFO [0] received reconcile message
Sep 22 23:22:24.779 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(545), op: ExtentClose { repair_id: ReconciliationId(545), extent_id: 100 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.779 INFO [0] client ExtentClose { repair_id: ReconciliationId(545), extent_id: 100 }
Sep 22 23:22:24.779 INFO [1] received reconcile message
Sep 22 23:22:24.779 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(545), op: ExtentClose { repair_id: ReconciliationId(545), extent_id: 100 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.779 INFO [1] client ExtentClose { repair_id: ReconciliationId(545), extent_id: 100 }
Sep 22 23:22:24.779 INFO [2] received reconcile message
Sep 22 23:22:24.779 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(545), op: ExtentClose { repair_id: ReconciliationId(545), extent_id: 100 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.779 INFO [2] client ExtentClose { repair_id: ReconciliationId(545), extent_id: 100 }
Sep 22 23:22:24.779 DEBG 545 Close extent 100
Sep 22 23:22:24.779 DEBG 545 Close extent 100
Sep 22 23:22:24.779 DEBG 545 Close extent 100
Sep 22 23:22:24.780 DEBG [2] It's time to notify for 545
Sep 22 23:22:24.780 INFO Completion from [2] id:545 status:true
Sep 22 23:22:24.780 INFO [546/752] Repair commands completed
Sep 22 23:22:24.780 INFO Pop front: ReconcileIO { id: ReconciliationId(546), op: ExtentRepair { repair_id: ReconciliationId(546), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.780 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.780 INFO [0] received reconcile message
Sep 22 23:22:24.780 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(546), op: ExtentRepair { repair_id: ReconciliationId(546), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.780 INFO [0] client ExtentRepair { repair_id: ReconciliationId(546), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.780 INFO [0] Sending repair request ReconciliationId(546)
Sep 22 23:22:24.780 INFO [1] received reconcile message
Sep 22 23:22:24.780 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(546), op: ExtentRepair { repair_id: ReconciliationId(546), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.780 INFO [1] client ExtentRepair { repair_id: ReconciliationId(546), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.780 INFO [1] No action required ReconciliationId(546)
Sep 22 23:22:24.780 INFO [2] received reconcile message
Sep 22 23:22:24.780 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(546), op: ExtentRepair { repair_id: ReconciliationId(546), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.780 INFO [2] client ExtentRepair { repair_id: ReconciliationId(546), extent_id: 100, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.780 INFO [2] No action required ReconciliationId(546)
Sep 22 23:22:24.780 DEBG 546 Repair extent 100 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.780 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/064.copy"
Sep 22 23:22:24.844 INFO accepted connection, remote_addr: 127.0.0.1:37092, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.845 TRCE incoming request, uri: /extent/100/files, method: GET, req_id: 54454a0e-21a2-4b20-9093-d83bf40e8d5c, remote_addr: 127.0.0.1:37092, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.845 INFO request completed, latency_us: 224, response_code: 200, uri: /extent/100/files, method: GET, req_id: 54454a0e-21a2-4b20-9093-d83bf40e8d5c, remote_addr: 127.0.0.1:37092, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.845 INFO eid:100 Found repair files: ["064", "064.db"]
Sep 22 23:22:24.845 TRCE incoming request, uri: /newextent/100/data, method: GET, req_id: 72b862f8-2500-4e85-b2fc-7d69bb1653c4, remote_addr: 127.0.0.1:37092, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.846 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/100/data, method: GET, req_id: 72b862f8-2500-4e85-b2fc-7d69bb1653c4, remote_addr: 127.0.0.1:37092, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.851 TRCE incoming request, uri: /newextent/100/db, method: GET, req_id: d50e9d04-de69-40e0-92ec-ec3e5391f5d5, remote_addr: 127.0.0.1:37092, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.851 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/100/db, method: GET, req_id: d50e9d04-de69-40e0-92ec-ec3e5391f5d5, remote_addr: 127.0.0.1:37092, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.852 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/064.copy" to "/tmp/downstairs-zrMnlo6G/00/000/064.replace"
Sep 22 23:22:24.852 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.853 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/064.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.853 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/064"
Sep 22 23:22:24.853 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/064.db"
Sep 22 23:22:24.853 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.853 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/064.replace" to "/tmp/downstairs-zrMnlo6G/00/000/064.completed"
Sep 22 23:22:24.853 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.853 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.854 DEBG [0] It's time to notify for 546
Sep 22 23:22:24.854 INFO Completion from [0] id:546 status:true
Sep 22 23:22:24.854 INFO [547/752] Repair commands completed
Sep 22 23:22:24.854 INFO Pop front: ReconcileIO { id: ReconciliationId(547), op: ExtentReopen { repair_id: ReconciliationId(547), extent_id: 100 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.854 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.854 INFO [0] received reconcile message
Sep 22 23:22:24.854 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(547), op: ExtentReopen { repair_id: ReconciliationId(547), extent_id: 100 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.854 INFO [0] client ExtentReopen { repair_id: ReconciliationId(547), extent_id: 100 }
Sep 22 23:22:24.854 INFO [1] received reconcile message
Sep 22 23:22:24.854 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(547), op: ExtentReopen { repair_id: ReconciliationId(547), extent_id: 100 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.854 INFO [1] client ExtentReopen { repair_id: ReconciliationId(547), extent_id: 100 }
Sep 22 23:22:24.854 INFO [2] received reconcile message
Sep 22 23:22:24.854 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(547), op: ExtentReopen { repair_id: ReconciliationId(547), extent_id: 100 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.854 INFO [2] client ExtentReopen { repair_id: ReconciliationId(547), extent_id: 100 }
Sep 22 23:22:24.854 DEBG 547 Reopen extent 100
Sep 22 23:22:24.855 DEBG 547 Reopen extent 100
Sep 22 23:22:24.855 DEBG 547 Reopen extent 100
Sep 22 23:22:24.856 DEBG [2] It's time to notify for 547
Sep 22 23:22:24.856 INFO Completion from [2] id:547 status:true
Sep 22 23:22:24.856 INFO [548/752] Repair commands completed
Sep 22 23:22:24.856 INFO Pop front: ReconcileIO { id: ReconciliationId(548), op: ExtentFlush { repair_id: ReconciliationId(548), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.856 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.856 INFO [0] received reconcile message
Sep 22 23:22:24.856 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(548), op: ExtentFlush { repair_id: ReconciliationId(548), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.856 INFO [0] client ExtentFlush { repair_id: ReconciliationId(548), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.856 INFO [1] received reconcile message
Sep 22 23:22:24.856 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(548), op: ExtentFlush { repair_id: ReconciliationId(548), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.856 INFO [1] client ExtentFlush { repair_id: ReconciliationId(548), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.856 INFO [2] received reconcile message
Sep 22 23:22:24.856 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(548), op: ExtentFlush { repair_id: ReconciliationId(548), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.856 INFO [2] client ExtentFlush { repair_id: ReconciliationId(548), extent_id: 95, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.856 DEBG 548 Flush extent 95 with f:2 g:2
Sep 22 23:22:24.857 DEBG Flush just extent 95 with f:2 and g:2
Sep 22 23:22:24.857 DEBG [1] It's time to notify for 548
Sep 22 23:22:24.857 INFO Completion from [1] id:548 status:true
Sep 22 23:22:24.857 INFO [549/752] Repair commands completed
Sep 22 23:22:24.857 INFO Pop front: ReconcileIO { id: ReconciliationId(549), op: ExtentClose { repair_id: ReconciliationId(549), extent_id: 95 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.857 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.857 INFO [0] received reconcile message
Sep 22 23:22:24.857 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(549), op: ExtentClose { repair_id: ReconciliationId(549), extent_id: 95 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.857 INFO [0] client ExtentClose { repair_id: ReconciliationId(549), extent_id: 95 }
Sep 22 23:22:24.857 INFO [1] received reconcile message
Sep 22 23:22:24.857 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(549), op: ExtentClose { repair_id: ReconciliationId(549), extent_id: 95 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.857 INFO [1] client ExtentClose { repair_id: ReconciliationId(549), extent_id: 95 }
Sep 22 23:22:24.857 INFO [2] received reconcile message
Sep 22 23:22:24.857 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(549), op: ExtentClose { repair_id: ReconciliationId(549), extent_id: 95 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.857 INFO [2] client ExtentClose { repair_id: ReconciliationId(549), extent_id: 95 }
Sep 22 23:22:24.857 DEBG 549 Close extent 95
Sep 22 23:22:24.857 DEBG 549 Close extent 95
Sep 22 23:22:24.858 DEBG 549 Close extent 95
Sep 22 23:22:24.858 DEBG [2] It's time to notify for 549
Sep 22 23:22:24.858 INFO Completion from [2] id:549 status:true
Sep 22 23:22:24.858 INFO [550/752] Repair commands completed
Sep 22 23:22:24.858 INFO Pop front: ReconcileIO { id: ReconciliationId(550), op: ExtentRepair { repair_id: ReconciliationId(550), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.858 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.858 INFO [0] received reconcile message
Sep 22 23:22:24.858 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(550), op: ExtentRepair { repair_id: ReconciliationId(550), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.858 INFO [0] client ExtentRepair { repair_id: ReconciliationId(550), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.858 INFO [0] Sending repair request ReconciliationId(550)
Sep 22 23:22:24.858 INFO [1] received reconcile message
Sep 22 23:22:24.858 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(550), op: ExtentRepair { repair_id: ReconciliationId(550), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.858 INFO [1] client ExtentRepair { repair_id: ReconciliationId(550), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.858 INFO [1] No action required ReconciliationId(550)
Sep 22 23:22:24.858 INFO [2] received reconcile message
Sep 22 23:22:24.858 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(550), op: ExtentRepair { repair_id: ReconciliationId(550), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.858 INFO [2] client ExtentRepair { repair_id: ReconciliationId(550), extent_id: 95, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.858 INFO [2] No action required ReconciliationId(550)
Sep 22 23:22:24.859 DEBG 550 Repair extent 95 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.859 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/05F.copy"
Sep 22 23:22:24.921 INFO accepted connection, remote_addr: 127.0.0.1:40895, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.922 TRCE incoming request, uri: /extent/95/files, method: GET, req_id: 72ac2774-d5d9-4530-84c1-f9f9a186815f, remote_addr: 127.0.0.1:40895, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.922 INFO request completed, latency_us: 202, response_code: 200, uri: /extent/95/files, method: GET, req_id: 72ac2774-d5d9-4530-84c1-f9f9a186815f, remote_addr: 127.0.0.1:40895, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.922 INFO eid:95 Found repair files: ["05F", "05F.db"]
Sep 22 23:22:24.922 TRCE incoming request, uri: /newextent/95/data, method: GET, req_id: 0f8c1b99-58a5-471f-b1e8-6c7764780bbc, remote_addr: 127.0.0.1:40895, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.923 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/95/data, method: GET, req_id: 0f8c1b99-58a5-471f-b1e8-6c7764780bbc, remote_addr: 127.0.0.1:40895, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.927 TRCE incoming request, uri: /newextent/95/db, method: GET, req_id: b9d6ff52-eec9-47df-88a5-278332912e22, remote_addr: 127.0.0.1:40895, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.928 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/95/db, method: GET, req_id: b9d6ff52-eec9-47df-88a5-278332912e22, remote_addr: 127.0.0.1:40895, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:24.929 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/05F.copy" to "/tmp/downstairs-zrMnlo6G/00/000/05F.replace"
Sep 22 23:22:24.929 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.930 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/05F.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.930 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05F"
Sep 22 23:22:24.930 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05F.db"
Sep 22 23:22:24.930 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.930 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/05F.replace" to "/tmp/downstairs-zrMnlo6G/00/000/05F.completed"
Sep 22 23:22:24.930 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.930 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:24.930 DEBG [0] It's time to notify for 550
Sep 22 23:22:24.931 INFO Completion from [0] id:550 status:true
Sep 22 23:22:24.931 INFO [551/752] Repair commands completed
Sep 22 23:22:24.931 INFO Pop front: ReconcileIO { id: ReconciliationId(551), op: ExtentReopen { repair_id: ReconciliationId(551), extent_id: 95 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.931 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.931 INFO [0] received reconcile message
Sep 22 23:22:24.931 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(551), op: ExtentReopen { repair_id: ReconciliationId(551), extent_id: 95 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.931 INFO [0] client ExtentReopen { repair_id: ReconciliationId(551), extent_id: 95 }
Sep 22 23:22:24.931 INFO [1] received reconcile message
Sep 22 23:22:24.931 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(551), op: ExtentReopen { repair_id: ReconciliationId(551), extent_id: 95 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.931 INFO [1] client ExtentReopen { repair_id: ReconciliationId(551), extent_id: 95 }
Sep 22 23:22:24.931 INFO [2] received reconcile message
Sep 22 23:22:24.931 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(551), op: ExtentReopen { repair_id: ReconciliationId(551), extent_id: 95 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.931 INFO [2] client ExtentReopen { repair_id: ReconciliationId(551), extent_id: 95 }
Sep 22 23:22:24.931 DEBG 551 Reopen extent 95
Sep 22 23:22:24.932 DEBG 551 Reopen extent 95
Sep 22 23:22:24.932 DEBG 551 Reopen extent 95
Sep 22 23:22:24.933 DEBG [2] It's time to notify for 551
Sep 22 23:22:24.933 INFO Completion from [2] id:551 status:true
Sep 22 23:22:24.933 INFO [552/752] Repair commands completed
Sep 22 23:22:24.933 INFO Pop front: ReconcileIO { id: ReconciliationId(552), op: ExtentFlush { repair_id: ReconciliationId(552), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.933 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.933 INFO [0] received reconcile message
Sep 22 23:22:24.933 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(552), op: ExtentFlush { repair_id: ReconciliationId(552), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.933 INFO [0] client ExtentFlush { repair_id: ReconciliationId(552), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.933 INFO [1] received reconcile message
Sep 22 23:22:24.933 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(552), op: ExtentFlush { repair_id: ReconciliationId(552), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:24.933 INFO [1] client ExtentFlush { repair_id: ReconciliationId(552), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.933 INFO [2] received reconcile message
Sep 22 23:22:24.933 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(552), op: ExtentFlush { repair_id: ReconciliationId(552), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.933 INFO [2] client ExtentFlush { repair_id: ReconciliationId(552), extent_id: 154, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:24.933 DEBG 552 Flush extent 154 with f:2 g:2
Sep 22 23:22:24.933 DEBG Flush just extent 154 with f:2 and g:2
Sep 22 23:22:24.933 DEBG [1] It's time to notify for 552
Sep 22 23:22:24.933 INFO Completion from [1] id:552 status:true
Sep 22 23:22:24.933 INFO [553/752] Repair commands completed
Sep 22 23:22:24.933 INFO Pop front: ReconcileIO { id: ReconciliationId(553), op: ExtentClose { repair_id: ReconciliationId(553), extent_id: 154 }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.933 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.933 INFO [0] received reconcile message
Sep 22 23:22:24.933 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(553), op: ExtentClose { repair_id: ReconciliationId(553), extent_id: 154 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.933 INFO [0] client ExtentClose { repair_id: ReconciliationId(553), extent_id: 154 }
Sep 22 23:22:24.933 INFO [1] received reconcile message
Sep 22 23:22:24.933 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(553), op: ExtentClose { repair_id: ReconciliationId(553), extent_id: 154 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.934 INFO [1] client ExtentClose { repair_id: ReconciliationId(553), extent_id: 154 }
Sep 22 23:22:24.934 INFO [2] received reconcile message
Sep 22 23:22:24.934 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(553), op: ExtentClose { repair_id: ReconciliationId(553), extent_id: 154 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:24.934 INFO [2] client ExtentClose { repair_id: ReconciliationId(553), extent_id: 154 }
Sep 22 23:22:24.934 DEBG 553 Close extent 154
Sep 22 23:22:24.934 DEBG 553 Close extent 154
Sep 22 23:22:24.934 DEBG 553 Close extent 154
Sep 22 23:22:24.935 DEBG [2] It's time to notify for 553
Sep 22 23:22:24.935 INFO Completion from [2] id:553 status:true
Sep 22 23:22:24.935 INFO [554/752] Repair commands completed
Sep 22 23:22:24.935 INFO Pop front: ReconcileIO { id: ReconciliationId(554), op: ExtentRepair { repair_id: ReconciliationId(554), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:24.935 INFO Sent repair work, now wait for resp
Sep 22 23:22:24.935 INFO [0] received reconcile message
Sep 22 23:22:24.935 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(554), op: ExtentRepair { repair_id: ReconciliationId(554), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:24.935 INFO [0] client ExtentRepair { repair_id: ReconciliationId(554), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.935 INFO [0] Sending repair request ReconciliationId(554)
Sep 22 23:22:24.935 INFO [1] received reconcile message
Sep 22 23:22:24.935 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(554), op: ExtentRepair { repair_id: ReconciliationId(554), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:24.935 INFO [1] client ExtentRepair { repair_id: ReconciliationId(554), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.935 INFO [1] No action required ReconciliationId(554)
Sep 22 23:22:24.935 INFO [2] received reconcile message
Sep 22 23:22:24.935 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(554), op: ExtentRepair { repair_id: ReconciliationId(554), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:24.935 INFO [2] client ExtentRepair { repair_id: ReconciliationId(554), extent_id: 154, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:24.935 INFO [2] No action required ReconciliationId(554)
Sep 22 23:22:24.935 DEBG 554 Repair extent 154 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:24.935 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/09A.copy"
Sep 22 23:22:25.000 INFO accepted connection, remote_addr: 127.0.0.1:35361, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.000 TRCE incoming request, uri: /extent/154/files, method: GET, req_id: aaa5a4ba-3718-4df6-b666-a6c949c53f90, remote_addr: 127.0.0.1:35361, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.000 INFO request completed, latency_us: 212, response_code: 200, uri: /extent/154/files, method: GET, req_id: aaa5a4ba-3718-4df6-b666-a6c949c53f90, remote_addr: 127.0.0.1:35361, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.000 INFO eid:154 Found repair files: ["09A", "09A.db"]
Sep 22 23:22:25.001 TRCE incoming request, uri: /newextent/154/data, method: GET, req_id: 5ec35068-0643-4975-a63c-2c90066ee389, remote_addr: 127.0.0.1:35361, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.001 INFO request completed, latency_us: 341, response_code: 200, uri: /newextent/154/data, method: GET, req_id: 5ec35068-0643-4975-a63c-2c90066ee389, remote_addr: 127.0.0.1:35361, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.006 TRCE incoming request, uri: /newextent/154/db, method: GET, req_id: 185946b0-4500-433f-9c10-4306b64b054a, remote_addr: 127.0.0.1:35361, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.006 INFO request completed, latency_us: 311, response_code: 200, uri: /newextent/154/db, method: GET, req_id: 185946b0-4500-433f-9c10-4306b64b054a, remote_addr: 127.0.0.1:35361, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.008 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/09A.copy" to "/tmp/downstairs-zrMnlo6G/00/000/09A.replace"
Sep 22 23:22:25.008 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.009 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/09A.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.009 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09A"
Sep 22 23:22:25.009 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09A.db"
Sep 22 23:22:25.009 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.009 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/09A.replace" to "/tmp/downstairs-zrMnlo6G/00/000/09A.completed"
Sep 22 23:22:25.009 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.009 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.010 DEBG [0] It's time to notify for 554
Sep 22 23:22:25.010 INFO Completion from [0] id:554 status:true
Sep 22 23:22:25.010 INFO [555/752] Repair commands completed
Sep 22 23:22:25.010 INFO Pop front: ReconcileIO { id: ReconciliationId(555), op: ExtentReopen { repair_id: ReconciliationId(555), extent_id: 154 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.010 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.010 INFO [0] received reconcile message
Sep 22 23:22:25.010 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(555), op: ExtentReopen { repair_id: ReconciliationId(555), extent_id: 154 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.010 INFO [0] client ExtentReopen { repair_id: ReconciliationId(555), extent_id: 154 }
Sep 22 23:22:25.010 INFO [1] received reconcile message
Sep 22 23:22:25.010 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(555), op: ExtentReopen { repair_id: ReconciliationId(555), extent_id: 154 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.010 INFO [1] client ExtentReopen { repair_id: ReconciliationId(555), extent_id: 154 }
Sep 22 23:22:25.010 INFO [2] received reconcile message
Sep 22 23:22:25.010 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(555), op: ExtentReopen { repair_id: ReconciliationId(555), extent_id: 154 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.010 INFO [2] client ExtentReopen { repair_id: ReconciliationId(555), extent_id: 154 }
Sep 22 23:22:25.010 DEBG 555 Reopen extent 154
Sep 22 23:22:25.011 DEBG 555 Reopen extent 154
Sep 22 23:22:25.012 DEBG 555 Reopen extent 154
Sep 22 23:22:25.012 DEBG [2] It's time to notify for 555
Sep 22 23:22:25.012 INFO Completion from [2] id:555 status:true
Sep 22 23:22:25.012 INFO [556/752] Repair commands completed
Sep 22 23:22:25.012 INFO Pop front: ReconcileIO { id: ReconciliationId(556), op: ExtentFlush { repair_id: ReconciliationId(556), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.012 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.012 INFO [0] received reconcile message
Sep 22 23:22:25.012 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(556), op: ExtentFlush { repair_id: ReconciliationId(556), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.012 INFO [0] client ExtentFlush { repair_id: ReconciliationId(556), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.013 INFO [1] received reconcile message
Sep 22 23:22:25.013 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(556), op: ExtentFlush { repair_id: ReconciliationId(556), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.013 INFO [1] client ExtentFlush { repair_id: ReconciliationId(556), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.013 INFO [2] received reconcile message
Sep 22 23:22:25.013 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(556), op: ExtentFlush { repair_id: ReconciliationId(556), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.013 INFO [2] client ExtentFlush { repair_id: ReconciliationId(556), extent_id: 50, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.013 DEBG 556 Flush extent 50 with f:2 g:2
Sep 22 23:22:25.013 DEBG Flush just extent 50 with f:2 and g:2
Sep 22 23:22:25.013 DEBG [1] It's time to notify for 556
Sep 22 23:22:25.013 INFO Completion from [1] id:556 status:true
Sep 22 23:22:25.013 INFO [557/752] Repair commands completed
Sep 22 23:22:25.013 INFO Pop front: ReconcileIO { id: ReconciliationId(557), op: ExtentClose { repair_id: ReconciliationId(557), extent_id: 50 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.013 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.013 INFO [0] received reconcile message
Sep 22 23:22:25.013 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(557), op: ExtentClose { repair_id: ReconciliationId(557), extent_id: 50 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.013 INFO [0] client ExtentClose { repair_id: ReconciliationId(557), extent_id: 50 }
Sep 22 23:22:25.013 INFO [1] received reconcile message
Sep 22 23:22:25.013 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(557), op: ExtentClose { repair_id: ReconciliationId(557), extent_id: 50 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.013 INFO [1] client ExtentClose { repair_id: ReconciliationId(557), extent_id: 50 }
Sep 22 23:22:25.013 INFO [2] received reconcile message
Sep 22 23:22:25.013 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(557), op: ExtentClose { repair_id: ReconciliationId(557), extent_id: 50 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.013 INFO [2] client ExtentClose { repair_id: ReconciliationId(557), extent_id: 50 }
Sep 22 23:22:25.013 DEBG 557 Close extent 50
Sep 22 23:22:25.014 DEBG 557 Close extent 50
Sep 22 23:22:25.014 DEBG 557 Close extent 50
Sep 22 23:22:25.014 DEBG [2] It's time to notify for 557
Sep 22 23:22:25.014 INFO Completion from [2] id:557 status:true
Sep 22 23:22:25.014 INFO [558/752] Repair commands completed
Sep 22 23:22:25.014 INFO Pop front: ReconcileIO { id: ReconciliationId(558), op: ExtentRepair { repair_id: ReconciliationId(558), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.014 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.014 INFO [0] received reconcile message
Sep 22 23:22:25.014 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(558), op: ExtentRepair { repair_id: ReconciliationId(558), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.014 INFO [0] client ExtentRepair { repair_id: ReconciliationId(558), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.015 INFO [0] Sending repair request ReconciliationId(558)
Sep 22 23:22:25.015 INFO [1] received reconcile message
Sep 22 23:22:25.015 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(558), op: ExtentRepair { repair_id: ReconciliationId(558), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.015 INFO [1] client ExtentRepair { repair_id: ReconciliationId(558), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.015 INFO [1] No action required ReconciliationId(558)
Sep 22 23:22:25.015 INFO [2] received reconcile message
Sep 22 23:22:25.015 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(558), op: ExtentRepair { repair_id: ReconciliationId(558), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.015 INFO [2] client ExtentRepair { repair_id: ReconciliationId(558), extent_id: 50, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.015 INFO [2] No action required ReconciliationId(558)
Sep 22 23:22:25.015 DEBG 558 Repair extent 50 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.015 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/032.copy"
Sep 22 23:22:25.079 INFO accepted connection, remote_addr: 127.0.0.1:43407, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.079 TRCE incoming request, uri: /extent/50/files, method: GET, req_id: 01233bc9-73b0-4caf-b5e0-32cdf2a6182b, remote_addr: 127.0.0.1:43407, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.079 INFO request completed, latency_us: 210, response_code: 200, uri: /extent/50/files, method: GET, req_id: 01233bc9-73b0-4caf-b5e0-32cdf2a6182b, remote_addr: 127.0.0.1:43407, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.080 INFO eid:50 Found repair files: ["032", "032.db"]
Sep 22 23:22:25.080 TRCE incoming request, uri: /newextent/50/data, method: GET, req_id: f73f8b44-9df8-4613-a0ae-a0a37cd3010b, remote_addr: 127.0.0.1:43407, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.080 INFO request completed, latency_us: 325, response_code: 200, uri: /newextent/50/data, method: GET, req_id: f73f8b44-9df8-4613-a0ae-a0a37cd3010b, remote_addr: 127.0.0.1:43407, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.085 TRCE incoming request, uri: /newextent/50/db, method: GET, req_id: dc1568cc-ee09-42d0-a67f-32e941131f78, remote_addr: 127.0.0.1:43407, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.085 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/50/db, method: GET, req_id: dc1568cc-ee09-42d0-a67f-32e941131f78, remote_addr: 127.0.0.1:43407, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.086 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/032.copy" to "/tmp/downstairs-zrMnlo6G/00/000/032.replace"
Sep 22 23:22:25.086 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.087 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/032.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.087 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/032"
Sep 22 23:22:25.087 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/032.db"
Sep 22 23:22:25.088 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.088 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/032.replace" to "/tmp/downstairs-zrMnlo6G/00/000/032.completed"
Sep 22 23:22:25.088 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.088 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.088 DEBG [0] It's time to notify for 558
Sep 22 23:22:25.088 INFO Completion from [0] id:558 status:true
Sep 22 23:22:25.088 INFO [559/752] Repair commands completed
Sep 22 23:22:25.088 INFO Pop front: ReconcileIO { id: ReconciliationId(559), op: ExtentReopen { repair_id: ReconciliationId(559), extent_id: 50 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.088 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.088 INFO [0] received reconcile message
Sep 22 23:22:25.088 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(559), op: ExtentReopen { repair_id: ReconciliationId(559), extent_id: 50 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.088 INFO [0] client ExtentReopen { repair_id: ReconciliationId(559), extent_id: 50 }
Sep 22 23:22:25.088 INFO [1] received reconcile message
Sep 22 23:22:25.088 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(559), op: ExtentReopen { repair_id: ReconciliationId(559), extent_id: 50 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.088 INFO [1] client ExtentReopen { repair_id: ReconciliationId(559), extent_id: 50 }
Sep 22 23:22:25.088 INFO [2] received reconcile message
Sep 22 23:22:25.088 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(559), op: ExtentReopen { repair_id: ReconciliationId(559), extent_id: 50 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.088 INFO [2] client ExtentReopen { repair_id: ReconciliationId(559), extent_id: 50 }
Sep 22 23:22:25.088 DEBG 559 Reopen extent 50
Sep 22 23:22:25.089 DEBG 559 Reopen extent 50
Sep 22 23:22:25.089 DEBG 559 Reopen extent 50
Sep 22 23:22:25.090 DEBG [2] It's time to notify for 559
Sep 22 23:22:25.090 INFO Completion from [2] id:559 status:true
Sep 22 23:22:25.090 INFO [560/752] Repair commands completed
Sep 22 23:22:25.090 INFO Pop front: ReconcileIO { id: ReconciliationId(560), op: ExtentFlush { repair_id: ReconciliationId(560), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.090 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.090 INFO [0] received reconcile message
Sep 22 23:22:25.090 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(560), op: ExtentFlush { repair_id: ReconciliationId(560), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.090 INFO [0] client ExtentFlush { repair_id: ReconciliationId(560), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.090 INFO [1] received reconcile message
Sep 22 23:22:25.090 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(560), op: ExtentFlush { repair_id: ReconciliationId(560), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.090 INFO [1] client ExtentFlush { repair_id: ReconciliationId(560), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.090 INFO [2] received reconcile message
Sep 22 23:22:25.090 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(560), op: ExtentFlush { repair_id: ReconciliationId(560), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.091 INFO [2] client ExtentFlush { repair_id: ReconciliationId(560), extent_id: 91, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.091 DEBG 560 Flush extent 91 with f:2 g:2
Sep 22 23:22:25.091 DEBG Flush just extent 91 with f:2 and g:2
Sep 22 23:22:25.091 DEBG [1] It's time to notify for 560
Sep 22 23:22:25.091 INFO Completion from [1] id:560 status:true
Sep 22 23:22:25.091 INFO [561/752] Repair commands completed
Sep 22 23:22:25.091 INFO Pop front: ReconcileIO { id: ReconciliationId(561), op: ExtentClose { repair_id: ReconciliationId(561), extent_id: 91 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.091 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.091 INFO [0] received reconcile message
Sep 22 23:22:25.091 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(561), op: ExtentClose { repair_id: ReconciliationId(561), extent_id: 91 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.091 INFO [0] client ExtentClose { repair_id: ReconciliationId(561), extent_id: 91 }
Sep 22 23:22:25.091 INFO [1] received reconcile message
Sep 22 23:22:25.091 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(561), op: ExtentClose { repair_id: ReconciliationId(561), extent_id: 91 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.091 INFO [1] client ExtentClose { repair_id: ReconciliationId(561), extent_id: 91 }
Sep 22 23:22:25.091 INFO [2] received reconcile message
Sep 22 23:22:25.091 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(561), op: ExtentClose { repair_id: ReconciliationId(561), extent_id: 91 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.091 INFO [2] client ExtentClose { repair_id: ReconciliationId(561), extent_id: 91 }
Sep 22 23:22:25.091 DEBG 561 Close extent 91
Sep 22 23:22:25.091 DEBG 561 Close extent 91
Sep 22 23:22:25.092 DEBG 561 Close extent 91
Sep 22 23:22:25.092 DEBG [2] It's time to notify for 561
Sep 22 23:22:25.092 INFO Completion from [2] id:561 status:true
Sep 22 23:22:25.092 INFO [562/752] Repair commands completed
Sep 22 23:22:25.092 INFO Pop front: ReconcileIO { id: ReconciliationId(562), op: ExtentRepair { repair_id: ReconciliationId(562), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.092 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.092 INFO [0] received reconcile message
Sep 22 23:22:25.092 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(562), op: ExtentRepair { repair_id: ReconciliationId(562), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.092 INFO [0] client ExtentRepair { repair_id: ReconciliationId(562), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.092 INFO [0] Sending repair request ReconciliationId(562)
Sep 22 23:22:25.092 INFO [1] received reconcile message
Sep 22 23:22:25.092 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(562), op: ExtentRepair { repair_id: ReconciliationId(562), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.092 INFO [1] client ExtentRepair { repair_id: ReconciliationId(562), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.093 INFO [1] No action required ReconciliationId(562)
Sep 22 23:22:25.093 INFO [2] received reconcile message
Sep 22 23:22:25.093 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(562), op: ExtentRepair { repair_id: ReconciliationId(562), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.093 INFO [2] client ExtentRepair { repair_id: ReconciliationId(562), extent_id: 91, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.093 INFO [2] No action required ReconciliationId(562)
Sep 22 23:22:25.093 DEBG 562 Repair extent 91 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.093 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/05B.copy"
Sep 22 23:22:25.116 DEBG [1] Read AckReady 1089, : downstairs
Sep 22 23:22:25.117 ERRO [1] job id 1090 saw error GenericError("test error")
Sep 22 23:22:25.118 DEBG up_ds_listen was notified
Sep 22 23:22:25.118 DEBG up_ds_listen process 1089
Sep 22 23:22:25.118 DEBG [A] ack job 1089:90, : downstairs
Sep 22 23:22:25.158 INFO accepted connection, remote_addr: 127.0.0.1:55866, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.159 TRCE incoming request, uri: /extent/91/files, method: GET, req_id: b382f27e-dea6-46a6-a6f2-9aefcceb9446, remote_addr: 127.0.0.1:55866, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.159 INFO request completed, latency_us: 204, response_code: 200, uri: /extent/91/files, method: GET, req_id: b382f27e-dea6-46a6-a6f2-9aefcceb9446, remote_addr: 127.0.0.1:55866, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.159 INFO eid:91 Found repair files: ["05B", "05B.db"]
Sep 22 23:22:25.160 TRCE incoming request, uri: /newextent/91/data, method: GET, req_id: a4131c46-b608-4804-b761-932fc0505706, remote_addr: 127.0.0.1:55866, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.160 INFO request completed, latency_us: 326, response_code: 200, uri: /newextent/91/data, method: GET, req_id: a4131c46-b608-4804-b761-932fc0505706, remote_addr: 127.0.0.1:55866, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.165 TRCE incoming request, uri: /newextent/91/db, method: GET, req_id: b679f97a-5bc0-434a-bd1b-ca948795f656, remote_addr: 127.0.0.1:55866, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.165 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/91/db, method: GET, req_id: b679f97a-5bc0-434a-bd1b-ca948795f656, remote_addr: 127.0.0.1:55866, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.166 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/05B.copy" to "/tmp/downstairs-zrMnlo6G/00/000/05B.replace"
Sep 22 23:22:25.166 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.167 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/05B.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.167 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05B"
Sep 22 23:22:25.167 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/05B.db"
Sep 22 23:22:25.167 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.167 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/05B.replace" to "/tmp/downstairs-zrMnlo6G/00/000/05B.completed"
Sep 22 23:22:25.167 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.167 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.168 DEBG [0] It's time to notify for 562
Sep 22 23:22:25.168 INFO Completion from [0] id:562 status:true
Sep 22 23:22:25.168 INFO [563/752] Repair commands completed
Sep 22 23:22:25.168 INFO Pop front: ReconcileIO { id: ReconciliationId(563), op: ExtentReopen { repair_id: ReconciliationId(563), extent_id: 91 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.168 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.168 INFO [0] received reconcile message
Sep 22 23:22:25.168 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(563), op: ExtentReopen { repair_id: ReconciliationId(563), extent_id: 91 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.168 INFO [0] client ExtentReopen { repair_id: ReconciliationId(563), extent_id: 91 }
Sep 22 23:22:25.168 INFO [1] received reconcile message
Sep 22 23:22:25.168 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(563), op: ExtentReopen { repair_id: ReconciliationId(563), extent_id: 91 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.168 INFO [1] client ExtentReopen { repair_id: ReconciliationId(563), extent_id: 91 }
Sep 22 23:22:25.168 INFO [2] received reconcile message
Sep 22 23:22:25.168 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(563), op: ExtentReopen { repair_id: ReconciliationId(563), extent_id: 91 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.168 INFO [2] client ExtentReopen { repair_id: ReconciliationId(563), extent_id: 91 }
Sep 22 23:22:25.168 DEBG 563 Reopen extent 91
Sep 22 23:22:25.169 DEBG 563 Reopen extent 91
Sep 22 23:22:25.169 DEBG 563 Reopen extent 91
Sep 22 23:22:25.170 DEBG [2] It's time to notify for 563
Sep 22 23:22:25.170 INFO Completion from [2] id:563 status:true
Sep 22 23:22:25.170 INFO [564/752] Repair commands completed
Sep 22 23:22:25.170 INFO Pop front: ReconcileIO { id: ReconciliationId(564), op: ExtentFlush { repair_id: ReconciliationId(564), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.170 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.170 INFO [0] received reconcile message
Sep 22 23:22:25.170 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(564), op: ExtentFlush { repair_id: ReconciliationId(564), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.170 INFO [0] client ExtentFlush { repair_id: ReconciliationId(564), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.170 INFO [1] received reconcile message
Sep 22 23:22:25.170 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(564), op: ExtentFlush { repair_id: ReconciliationId(564), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.170 INFO [1] client ExtentFlush { repair_id: ReconciliationId(564), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.170 INFO [2] received reconcile message
Sep 22 23:22:25.170 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(564), op: ExtentFlush { repair_id: ReconciliationId(564), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.170 INFO [2] client ExtentFlush { repair_id: ReconciliationId(564), extent_id: 158, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.170 DEBG 564 Flush extent 158 with f:2 g:2
Sep 22 23:22:25.170 DEBG Flush just extent 158 with f:2 and g:2
Sep 22 23:22:25.170 DEBG [1] It's time to notify for 564
Sep 22 23:22:25.170 INFO Completion from [1] id:564 status:true
Sep 22 23:22:25.170 INFO [565/752] Repair commands completed
Sep 22 23:22:25.171 INFO Pop front: ReconcileIO { id: ReconciliationId(565), op: ExtentClose { repair_id: ReconciliationId(565), extent_id: 158 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.171 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.171 INFO [0] received reconcile message
Sep 22 23:22:25.171 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(565), op: ExtentClose { repair_id: ReconciliationId(565), extent_id: 158 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.171 INFO [0] client ExtentClose { repair_id: ReconciliationId(565), extent_id: 158 }
Sep 22 23:22:25.171 INFO [1] received reconcile message
Sep 22 23:22:25.171 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(565), op: ExtentClose { repair_id: ReconciliationId(565), extent_id: 158 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.171 INFO [1] client ExtentClose { repair_id: ReconciliationId(565), extent_id: 158 }
Sep 22 23:22:25.171 INFO [2] received reconcile message
Sep 22 23:22:25.171 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(565), op: ExtentClose { repair_id: ReconciliationId(565), extent_id: 158 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.171 INFO [2] client ExtentClose { repair_id: ReconciliationId(565), extent_id: 158 }
Sep 22 23:22:25.171 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:25.171 DEBG 565 Close extent 158
Sep 22 23:22:25.171 DEBG 565 Close extent 158
Sep 22 23:22:25.171 DEBG 565 Close extent 158
Sep 22 23:22:25.172 DEBG [2] It's time to notify for 565
Sep 22 23:22:25.172 INFO Completion from [2] id:565 status:true
Sep 22 23:22:25.172 INFO [566/752] Repair commands completed
Sep 22 23:22:25.172 INFO Pop front: ReconcileIO { id: ReconciliationId(566), op: ExtentRepair { repair_id: ReconciliationId(566), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.172 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.172 INFO [0] received reconcile message
Sep 22 23:22:25.172 WARN returning error on read!
Sep 22 23:22:25.172 DEBG Read      :1089 deps:[JobId(1088)] res:false
Sep 22 23:22:25.172 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(566), op: ExtentRepair { repair_id: ReconciliationId(566), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.172 INFO [0] client ExtentRepair { repair_id: ReconciliationId(566), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.172 INFO [0] Sending repair request ReconciliationId(566)
Sep 22 23:22:25.172 INFO [1] received reconcile message
Sep 22 23:22:25.172 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(566), op: ExtentRepair { repair_id: ReconciliationId(566), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.172 INFO [1] client ExtentRepair { repair_id: ReconciliationId(566), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.172 INFO [1] No action required ReconciliationId(566)
Sep 22 23:22:25.172 INFO [2] received reconcile message
Sep 22 23:22:25.172 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(566), op: ExtentRepair { repair_id: ReconciliationId(566), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.172 INFO [2] client ExtentRepair { repair_id: ReconciliationId(566), extent_id: 158, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.172 INFO [2] No action required ReconciliationId(566)
Sep 22 23:22:25.172 DEBG 566 Repair extent 158 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.172 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/09E.copy"
Sep 22 23:22:25.178 DEBG Read      :1089 deps:[JobId(1088)] res:true
Sep 22 23:22:25.199 DEBG IO Read  1091 has deps [JobId(1090)]
Sep 22 23:22:25.200 ERRO [0] job id 1089 saw error GenericError("test error")
Sep 22 23:22:25.213 DEBG Flush     :1090 extent_limit None deps:[JobId(1089), JobId(1088)] res:true f:34 g:1
Sep 22 23:22:25.219 DEBG Read      :1091 deps:[JobId(1090)] res:true
Sep 22 23:22:25.238 INFO accepted connection, remote_addr: 127.0.0.1:44139, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.238 TRCE incoming request, uri: /extent/158/files, method: GET, req_id: 7d4fce06-12bc-4138-ae19-3297800b269d, remote_addr: 127.0.0.1:44139, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.238 INFO request completed, latency_us: 212, response_code: 200, uri: /extent/158/files, method: GET, req_id: 7d4fce06-12bc-4138-ae19-3297800b269d, remote_addr: 127.0.0.1:44139, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.239 INFO eid:158 Found repair files: ["09E", "09E.db"]
Sep 22 23:22:25.239 TRCE incoming request, uri: /newextent/158/data, method: GET, req_id: 94d36839-fc84-4d10-8dda-d113b8c43bb0, remote_addr: 127.0.0.1:44139, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.239 INFO request completed, latency_us: 333, response_code: 200, uri: /newextent/158/data, method: GET, req_id: 94d36839-fc84-4d10-8dda-d113b8c43bb0, remote_addr: 127.0.0.1:44139, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.244 TRCE incoming request, uri: /newextent/158/db, method: GET, req_id: 9c1e0009-c6b4-4c16-b8e8-0a61bc1848a2, remote_addr: 127.0.0.1:44139, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.244 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/158/db, method: GET, req_id: 9c1e0009-c6b4-4c16-b8e8-0a61bc1848a2, remote_addr: 127.0.0.1:44139, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.245 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/09E.copy" to "/tmp/downstairs-zrMnlo6G/00/000/09E.replace"
Sep 22 23:22:25.245 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.246 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/09E.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.246 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09E"
Sep 22 23:22:25.246 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09E.db"
Sep 22 23:22:25.246 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.246 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/09E.replace" to "/tmp/downstairs-zrMnlo6G/00/000/09E.completed"
Sep 22 23:22:25.246 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.247 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.247 DEBG [0] It's time to notify for 566
Sep 22 23:22:25.247 INFO Completion from [0] id:566 status:true
Sep 22 23:22:25.247 INFO [567/752] Repair commands completed
Sep 22 23:22:25.247 INFO Pop front: ReconcileIO { id: ReconciliationId(567), op: ExtentReopen { repair_id: ReconciliationId(567), extent_id: 158 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.247 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.247 INFO [0] received reconcile message
Sep 22 23:22:25.247 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(567), op: ExtentReopen { repair_id: ReconciliationId(567), extent_id: 158 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.247 INFO [0] client ExtentReopen { repair_id: ReconciliationId(567), extent_id: 158 }
Sep 22 23:22:25.247 INFO [1] received reconcile message
Sep 22 23:22:25.247 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(567), op: ExtentReopen { repair_id: ReconciliationId(567), extent_id: 158 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.247 INFO [1] client ExtentReopen { repair_id: ReconciliationId(567), extent_id: 158 }
Sep 22 23:22:25.247 INFO [2] received reconcile message
Sep 22 23:22:25.247 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(567), op: ExtentReopen { repair_id: ReconciliationId(567), extent_id: 158 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.247 INFO [2] client ExtentReopen { repair_id: ReconciliationId(567), extent_id: 158 }
Sep 22 23:22:25.247 DEBG 567 Reopen extent 158
Sep 22 23:22:25.248 DEBG 567 Reopen extent 158
Sep 22 23:22:25.248 DEBG 567 Reopen extent 158
Sep 22 23:22:25.249 DEBG [2] It's time to notify for 567
Sep 22 23:22:25.249 INFO Completion from [2] id:567 status:true
Sep 22 23:22:25.249 INFO [568/752] Repair commands completed
Sep 22 23:22:25.249 INFO Pop front: ReconcileIO { id: ReconciliationId(568), op: ExtentFlush { repair_id: ReconciliationId(568), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.249 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.249 INFO [0] received reconcile message
Sep 22 23:22:25.249 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(568), op: ExtentFlush { repair_id: ReconciliationId(568), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.249 INFO [0] client ExtentFlush { repair_id: ReconciliationId(568), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.249 INFO [1] received reconcile message
Sep 22 23:22:25.249 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(568), op: ExtentFlush { repair_id: ReconciliationId(568), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.249 INFO [1] client ExtentFlush { repair_id: ReconciliationId(568), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.249 INFO [2] received reconcile message
Sep 22 23:22:25.249 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(568), op: ExtentFlush { repair_id: ReconciliationId(568), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.249 INFO [2] client ExtentFlush { repair_id: ReconciliationId(568), extent_id: 177, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.250 DEBG 568 Flush extent 177 with f:2 g:2
Sep 22 23:22:25.250 DEBG Flush just extent 177 with f:2 and g:2
Sep 22 23:22:25.250 DEBG [1] It's time to notify for 568
Sep 22 23:22:25.250 INFO Completion from [1] id:568 status:true
Sep 22 23:22:25.250 INFO [569/752] Repair commands completed
Sep 22 23:22:25.250 INFO Pop front: ReconcileIO { id: ReconciliationId(569), op: ExtentClose { repair_id: ReconciliationId(569), extent_id: 177 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.250 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.250 INFO [0] received reconcile message
Sep 22 23:22:25.250 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(569), op: ExtentClose { repair_id: ReconciliationId(569), extent_id: 177 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.250 INFO [0] client ExtentClose { repair_id: ReconciliationId(569), extent_id: 177 }
Sep 22 23:22:25.250 INFO [1] received reconcile message
Sep 22 23:22:25.250 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(569), op: ExtentClose { repair_id: ReconciliationId(569), extent_id: 177 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.250 INFO [1] client ExtentClose { repair_id: ReconciliationId(569), extent_id: 177 }
Sep 22 23:22:25.250 INFO [2] received reconcile message
Sep 22 23:22:25.250 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(569), op: ExtentClose { repair_id: ReconciliationId(569), extent_id: 177 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.250 INFO [2] client ExtentClose { repair_id: ReconciliationId(569), extent_id: 177 }
Sep 22 23:22:25.250 DEBG 569 Close extent 177
Sep 22 23:22:25.250 DEBG 569 Close extent 177
Sep 22 23:22:25.251 DEBG 569 Close extent 177
Sep 22 23:22:25.251 DEBG [2] It's time to notify for 569
Sep 22 23:22:25.251 INFO Completion from [2] id:569 status:true
Sep 22 23:22:25.251 INFO [570/752] Repair commands completed
Sep 22 23:22:25.251 INFO Pop front: ReconcileIO { id: ReconciliationId(570), op: ExtentRepair { repair_id: ReconciliationId(570), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.251 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.251 INFO [0] received reconcile message
Sep 22 23:22:25.251 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(570), op: ExtentRepair { repair_id: ReconciliationId(570), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.251 INFO [0] client ExtentRepair { repair_id: ReconciliationId(570), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.251 INFO [0] Sending repair request ReconciliationId(570)
Sep 22 23:22:25.251 INFO [1] received reconcile message
Sep 22 23:22:25.251 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(570), op: ExtentRepair { repair_id: ReconciliationId(570), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.251 INFO [1] client ExtentRepair { repair_id: ReconciliationId(570), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.251 INFO [1] No action required ReconciliationId(570)
Sep 22 23:22:25.251 INFO [2] received reconcile message
Sep 22 23:22:25.251 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(570), op: ExtentRepair { repair_id: ReconciliationId(570), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.251 INFO [2] client ExtentRepair { repair_id: ReconciliationId(570), extent_id: 177, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.252 INFO [2] No action required ReconciliationId(570)
Sep 22 23:22:25.252 DEBG 570 Repair extent 177 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.252 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0B1.copy"
Sep 22 23:22:25.316 INFO accepted connection, remote_addr: 127.0.0.1:62403, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.317 TRCE incoming request, uri: /extent/177/files, method: GET, req_id: a71e632f-2612-4078-915c-6d53daa1c12f, remote_addr: 127.0.0.1:62403, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.317 INFO request completed, latency_us: 215, response_code: 200, uri: /extent/177/files, method: GET, req_id: a71e632f-2612-4078-915c-6d53daa1c12f, remote_addr: 127.0.0.1:62403, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.317 INFO eid:177 Found repair files: ["0B1", "0B1.db"]
Sep 22 23:22:25.317 TRCE incoming request, uri: /newextent/177/data, method: GET, req_id: 51b99753-2242-44c4-b420-e3ec4a8fcc8a, remote_addr: 127.0.0.1:62403, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.318 INFO request completed, latency_us: 318, response_code: 200, uri: /newextent/177/data, method: GET, req_id: 51b99753-2242-44c4-b420-e3ec4a8fcc8a, remote_addr: 127.0.0.1:62403, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.323 TRCE incoming request, uri: /newextent/177/db, method: GET, req_id: f1a0743b-5c53-40d5-9a2d-49d7d18e5627, remote_addr: 127.0.0.1:62403, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.323 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/177/db, method: GET, req_id: f1a0743b-5c53-40d5-9a2d-49d7d18e5627, remote_addr: 127.0.0.1:62403, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.324 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0B1.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0B1.replace"
Sep 22 23:22:25.324 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.325 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0B1.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.325 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B1"
Sep 22 23:22:25.325 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B1.db"
Sep 22 23:22:25.325 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.325 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0B1.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0B1.completed"
Sep 22 23:22:25.325 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.325 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.326 DEBG [0] It's time to notify for 570
Sep 22 23:22:25.326 INFO Completion from [0] id:570 status:true
Sep 22 23:22:25.326 INFO [571/752] Repair commands completed
Sep 22 23:22:25.326 INFO Pop front: ReconcileIO { id: ReconciliationId(571), op: ExtentReopen { repair_id: ReconciliationId(571), extent_id: 177 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.326 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.326 INFO [0] received reconcile message
Sep 22 23:22:25.326 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(571), op: ExtentReopen { repair_id: ReconciliationId(571), extent_id: 177 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.326 INFO [0] client ExtentReopen { repair_id: ReconciliationId(571), extent_id: 177 }
Sep 22 23:22:25.326 INFO [1] received reconcile message
Sep 22 23:22:25.326 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(571), op: ExtentReopen { repair_id: ReconciliationId(571), extent_id: 177 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.326 INFO [1] client ExtentReopen { repair_id: ReconciliationId(571), extent_id: 177 }
Sep 22 23:22:25.326 INFO [2] received reconcile message
Sep 22 23:22:25.326 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(571), op: ExtentReopen { repair_id: ReconciliationId(571), extent_id: 177 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.326 INFO [2] client ExtentReopen { repair_id: ReconciliationId(571), extent_id: 177 }
Sep 22 23:22:25.326 DEBG 571 Reopen extent 177
Sep 22 23:22:25.327 DEBG 571 Reopen extent 177
Sep 22 23:22:25.327 DEBG 571 Reopen extent 177
Sep 22 23:22:25.328 DEBG [2] It's time to notify for 571
Sep 22 23:22:25.328 INFO Completion from [2] id:571 status:true
Sep 22 23:22:25.328 INFO [572/752] Repair commands completed
Sep 22 23:22:25.328 INFO Pop front: ReconcileIO { id: ReconciliationId(572), op: ExtentFlush { repair_id: ReconciliationId(572), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.328 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.328 INFO [0] received reconcile message
Sep 22 23:22:25.328 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(572), op: ExtentFlush { repair_id: ReconciliationId(572), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.328 INFO [0] client ExtentFlush { repair_id: ReconciliationId(572), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.328 INFO [1] received reconcile message
Sep 22 23:22:25.328 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(572), op: ExtentFlush { repair_id: ReconciliationId(572), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.328 INFO [1] client ExtentFlush { repair_id: ReconciliationId(572), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.328 INFO [2] received reconcile message
Sep 22 23:22:25.328 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(572), op: ExtentFlush { repair_id: ReconciliationId(572), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.328 INFO [2] client ExtentFlush { repair_id: ReconciliationId(572), extent_id: 134, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.328 DEBG 572 Flush extent 134 with f:2 g:2
Sep 22 23:22:25.328 DEBG Flush just extent 134 with f:2 and g:2
Sep 22 23:22:25.328 DEBG [1] It's time to notify for 572
Sep 22 23:22:25.328 INFO Completion from [1] id:572 status:true
Sep 22 23:22:25.329 INFO [573/752] Repair commands completed
Sep 22 23:22:25.329 INFO Pop front: ReconcileIO { id: ReconciliationId(573), op: ExtentClose { repair_id: ReconciliationId(573), extent_id: 134 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.329 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.329 INFO [0] received reconcile message
Sep 22 23:22:25.329 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(573), op: ExtentClose { repair_id: ReconciliationId(573), extent_id: 134 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.329 INFO [0] client ExtentClose { repair_id: ReconciliationId(573), extent_id: 134 }
Sep 22 23:22:25.329 INFO [1] received reconcile message
Sep 22 23:22:25.329 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(573), op: ExtentClose { repair_id: ReconciliationId(573), extent_id: 134 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.329 INFO [1] client ExtentClose { repair_id: ReconciliationId(573), extent_id: 134 }
Sep 22 23:22:25.329 INFO [2] received reconcile message
Sep 22 23:22:25.329 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(573), op: ExtentClose { repair_id: ReconciliationId(573), extent_id: 134 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.329 INFO [2] client ExtentClose { repair_id: ReconciliationId(573), extent_id: 134 }
Sep 22 23:22:25.329 DEBG 573 Close extent 134
Sep 22 23:22:25.329 DEBG 573 Close extent 134
Sep 22 23:22:25.329 DEBG 573 Close extent 134
Sep 22 23:22:25.330 DEBG [2] It's time to notify for 573
Sep 22 23:22:25.330 INFO Completion from [2] id:573 status:true
Sep 22 23:22:25.330 INFO [574/752] Repair commands completed
Sep 22 23:22:25.330 INFO Pop front: ReconcileIO { id: ReconciliationId(574), op: ExtentRepair { repair_id: ReconciliationId(574), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.330 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.330 INFO [0] received reconcile message
Sep 22 23:22:25.330 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(574), op: ExtentRepair { repair_id: ReconciliationId(574), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.330 INFO [0] client ExtentRepair { repair_id: ReconciliationId(574), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.330 INFO [0] Sending repair request ReconciliationId(574)
Sep 22 23:22:25.330 INFO [1] received reconcile message
Sep 22 23:22:25.330 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(574), op: ExtentRepair { repair_id: ReconciliationId(574), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.330 INFO [1] client ExtentRepair { repair_id: ReconciliationId(574), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.330 INFO [1] No action required ReconciliationId(574)
Sep 22 23:22:25.330 INFO [2] received reconcile message
Sep 22 23:22:25.330 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(574), op: ExtentRepair { repair_id: ReconciliationId(574), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.330 INFO [2] client ExtentRepair { repair_id: ReconciliationId(574), extent_id: 134, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.330 INFO [2] No action required ReconciliationId(574)
Sep 22 23:22:25.330 DEBG 574 Repair extent 134 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.330 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/086.copy"
Sep 22 23:22:25.395 INFO accepted connection, remote_addr: 127.0.0.1:53520, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.395 TRCE incoming request, uri: /extent/134/files, method: GET, req_id: d5f937a2-74d8-47da-a4da-a4f5dbb0a0f7, remote_addr: 127.0.0.1:53520, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.395 INFO request completed, latency_us: 201, response_code: 200, uri: /extent/134/files, method: GET, req_id: d5f937a2-74d8-47da-a4da-a4f5dbb0a0f7, remote_addr: 127.0.0.1:53520, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.396 INFO eid:134 Found repair files: ["086", "086.db"]
Sep 22 23:22:25.396 TRCE incoming request, uri: /newextent/134/data, method: GET, req_id: f1bf4896-b15a-486f-abf3-376e80d8463d, remote_addr: 127.0.0.1:53520, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.396 INFO request completed, latency_us: 322, response_code: 200, uri: /newextent/134/data, method: GET, req_id: f1bf4896-b15a-486f-abf3-376e80d8463d, remote_addr: 127.0.0.1:53520, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.401 TRCE incoming request, uri: /newextent/134/db, method: GET, req_id: e7b9a11e-fec2-4ec4-a46c-275d0a912cbe, remote_addr: 127.0.0.1:53520, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.401 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/134/db, method: GET, req_id: e7b9a11e-fec2-4ec4-a46c-275d0a912cbe, remote_addr: 127.0.0.1:53520, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.402 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/086.copy" to "/tmp/downstairs-zrMnlo6G/00/000/086.replace"
Sep 22 23:22:25.402 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.403 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/086.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.403 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/086"
Sep 22 23:22:25.403 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/086.db"
Sep 22 23:22:25.403 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.403 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/086.replace" to "/tmp/downstairs-zrMnlo6G/00/000/086.completed"
Sep 22 23:22:25.403 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.404 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.404 DEBG [0] It's time to notify for 574
Sep 22 23:22:25.404 INFO Completion from [0] id:574 status:true
Sep 22 23:22:25.404 INFO [575/752] Repair commands completed
Sep 22 23:22:25.404 INFO Pop front: ReconcileIO { id: ReconciliationId(575), op: ExtentReopen { repair_id: ReconciliationId(575), extent_id: 134 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.404 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.404 INFO [0] received reconcile message
Sep 22 23:22:25.404 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(575), op: ExtentReopen { repair_id: ReconciliationId(575), extent_id: 134 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.404 INFO [0] client ExtentReopen { repair_id: ReconciliationId(575), extent_id: 134 }
Sep 22 23:22:25.404 INFO [1] received reconcile message
Sep 22 23:22:25.404 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(575), op: ExtentReopen { repair_id: ReconciliationId(575), extent_id: 134 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.404 INFO [1] client ExtentReopen { repair_id: ReconciliationId(575), extent_id: 134 }
Sep 22 23:22:25.404 INFO [2] received reconcile message
Sep 22 23:22:25.404 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(575), op: ExtentReopen { repair_id: ReconciliationId(575), extent_id: 134 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.404 INFO [2] client ExtentReopen { repair_id: ReconciliationId(575), extent_id: 134 }
Sep 22 23:22:25.404 DEBG 575 Reopen extent 134
Sep 22 23:22:25.405 DEBG 575 Reopen extent 134
Sep 22 23:22:25.405 DEBG 575 Reopen extent 134
Sep 22 23:22:25.406 DEBG [2] It's time to notify for 575
Sep 22 23:22:25.406 INFO Completion from [2] id:575 status:true
Sep 22 23:22:25.406 INFO [576/752] Repair commands completed
Sep 22 23:22:25.406 INFO Pop front: ReconcileIO { id: ReconciliationId(576), op: ExtentFlush { repair_id: ReconciliationId(576), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.406 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.406 INFO [0] received reconcile message
Sep 22 23:22:25.406 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(576), op: ExtentFlush { repair_id: ReconciliationId(576), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.406 INFO [0] client ExtentFlush { repair_id: ReconciliationId(576), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.406 INFO [1] received reconcile message
Sep 22 23:22:25.406 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(576), op: ExtentFlush { repair_id: ReconciliationId(576), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.406 INFO [1] client ExtentFlush { repair_id: ReconciliationId(576), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.406 INFO [2] received reconcile message
Sep 22 23:22:25.406 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(576), op: ExtentFlush { repair_id: ReconciliationId(576), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.406 INFO [2] client ExtentFlush { repair_id: ReconciliationId(576), extent_id: 69, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.406 DEBG 576 Flush extent 69 with f:2 g:2
Sep 22 23:22:25.406 DEBG Flush just extent 69 with f:2 and g:2
Sep 22 23:22:25.407 DEBG [1] It's time to notify for 576
Sep 22 23:22:25.407 INFO Completion from [1] id:576 status:true
Sep 22 23:22:25.407 INFO [577/752] Repair commands completed
Sep 22 23:22:25.407 INFO Pop front: ReconcileIO { id: ReconciliationId(577), op: ExtentClose { repair_id: ReconciliationId(577), extent_id: 69 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.407 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.407 INFO [0] received reconcile message
Sep 22 23:22:25.407 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(577), op: ExtentClose { repair_id: ReconciliationId(577), extent_id: 69 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.407 INFO [0] client ExtentClose { repair_id: ReconciliationId(577), extent_id: 69 }
Sep 22 23:22:25.407 INFO [1] received reconcile message
Sep 22 23:22:25.407 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(577), op: ExtentClose { repair_id: ReconciliationId(577), extent_id: 69 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.407 INFO [1] client ExtentClose { repair_id: ReconciliationId(577), extent_id: 69 }
Sep 22 23:22:25.407 INFO [2] received reconcile message
Sep 22 23:22:25.407 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(577), op: ExtentClose { repair_id: ReconciliationId(577), extent_id: 69 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.407 INFO [2] client ExtentClose { repair_id: ReconciliationId(577), extent_id: 69 }
Sep 22 23:22:25.407 DEBG 577 Close extent 69
Sep 22 23:22:25.407 DEBG 577 Close extent 69
Sep 22 23:22:25.408 DEBG 577 Close extent 69
Sep 22 23:22:25.408 DEBG [2] It's time to notify for 577
Sep 22 23:22:25.408 INFO Completion from [2] id:577 status:true
Sep 22 23:22:25.408 INFO [578/752] Repair commands completed
Sep 22 23:22:25.408 INFO Pop front: ReconcileIO { id: ReconciliationId(578), op: ExtentRepair { repair_id: ReconciliationId(578), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.408 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.408 INFO [0] received reconcile message
Sep 22 23:22:25.408 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(578), op: ExtentRepair { repair_id: ReconciliationId(578), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.408 INFO [0] client ExtentRepair { repair_id: ReconciliationId(578), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.408 INFO [0] Sending repair request ReconciliationId(578)
Sep 22 23:22:25.408 INFO [1] received reconcile message
Sep 22 23:22:25.408 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(578), op: ExtentRepair { repair_id: ReconciliationId(578), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.408 INFO [1] client ExtentRepair { repair_id: ReconciliationId(578), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.408 INFO [1] No action required ReconciliationId(578)
Sep 22 23:22:25.408 INFO [2] received reconcile message
Sep 22 23:22:25.408 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(578), op: ExtentRepair { repair_id: ReconciliationId(578), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.408 INFO [2] client ExtentRepair { repair_id: ReconciliationId(578), extent_id: 69, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.408 INFO [2] No action required ReconciliationId(578)
Sep 22 23:22:25.409 DEBG 578 Repair extent 69 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.409 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/045.copy"
Sep 22 23:22:25.475 INFO accepted connection, remote_addr: 127.0.0.1:51898, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.475 TRCE incoming request, uri: /extent/69/files, method: GET, req_id: 992447ea-399c-4a38-b230-8560abc1f62b, remote_addr: 127.0.0.1:51898, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.475 INFO request completed, latency_us: 268, response_code: 200, uri: /extent/69/files, method: GET, req_id: 992447ea-399c-4a38-b230-8560abc1f62b, remote_addr: 127.0.0.1:51898, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.476 INFO eid:69 Found repair files: ["045", "045.db"]
Sep 22 23:22:25.476 TRCE incoming request, uri: /newextent/69/data, method: GET, req_id: 5e02e57a-58c4-4653-b377-d77e8d6acaa3, remote_addr: 127.0.0.1:51898, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.476 INFO request completed, latency_us: 345, response_code: 200, uri: /newextent/69/data, method: GET, req_id: 5e02e57a-58c4-4653-b377-d77e8d6acaa3, remote_addr: 127.0.0.1:51898, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.482 TRCE incoming request, uri: /newextent/69/db, method: GET, req_id: 7ff78441-283b-4fe3-9760-28a32ef94c13, remote_addr: 127.0.0.1:51898, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.482 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/69/db, method: GET, req_id: 7ff78441-283b-4fe3-9760-28a32ef94c13, remote_addr: 127.0.0.1:51898, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.483 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/045.copy" to "/tmp/downstairs-zrMnlo6G/00/000/045.replace"
Sep 22 23:22:25.483 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.484 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/045.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.484 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/045"
Sep 22 23:22:25.485 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/045.db"
Sep 22 23:22:25.485 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.485 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/045.replace" to "/tmp/downstairs-zrMnlo6G/00/000/045.completed"
Sep 22 23:22:25.485 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.485 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.485 DEBG [0] It's time to notify for 578
Sep 22 23:22:25.485 INFO Completion from [0] id:578 status:true
Sep 22 23:22:25.485 INFO [579/752] Repair commands completed
Sep 22 23:22:25.485 INFO Pop front: ReconcileIO { id: ReconciliationId(579), op: ExtentReopen { repair_id: ReconciliationId(579), extent_id: 69 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.485 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.485 INFO [0] received reconcile message
Sep 22 23:22:25.485 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(579), op: ExtentReopen { repair_id: ReconciliationId(579), extent_id: 69 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.485 INFO [0] client ExtentReopen { repair_id: ReconciliationId(579), extent_id: 69 }
Sep 22 23:22:25.485 INFO [1] received reconcile message
Sep 22 23:22:25.485 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(579), op: ExtentReopen { repair_id: ReconciliationId(579), extent_id: 69 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.485 INFO [1] client ExtentReopen { repair_id: ReconciliationId(579), extent_id: 69 }
Sep 22 23:22:25.485 INFO [2] received reconcile message
Sep 22 23:22:25.485 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(579), op: ExtentReopen { repair_id: ReconciliationId(579), extent_id: 69 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.485 INFO [2] client ExtentReopen { repair_id: ReconciliationId(579), extent_id: 69 }
Sep 22 23:22:25.486 DEBG 579 Reopen extent 69
Sep 22 23:22:25.486 DEBG 579 Reopen extent 69
Sep 22 23:22:25.487 DEBG 579 Reopen extent 69
Sep 22 23:22:25.488 DEBG [2] It's time to notify for 579
Sep 22 23:22:25.488 INFO Completion from [2] id:579 status:true
Sep 22 23:22:25.488 INFO [580/752] Repair commands completed
Sep 22 23:22:25.488 INFO Pop front: ReconcileIO { id: ReconciliationId(580), op: ExtentFlush { repair_id: ReconciliationId(580), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.488 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.488 INFO [0] received reconcile message
Sep 22 23:22:25.488 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(580), op: ExtentFlush { repair_id: ReconciliationId(580), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.488 INFO [0] client ExtentFlush { repair_id: ReconciliationId(580), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.488 INFO [1] received reconcile message
Sep 22 23:22:25.488 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(580), op: ExtentFlush { repair_id: ReconciliationId(580), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.488 INFO [1] client ExtentFlush { repair_id: ReconciliationId(580), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.488 INFO [2] received reconcile message
Sep 22 23:22:25.488 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(580), op: ExtentFlush { repair_id: ReconciliationId(580), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.488 INFO [2] client ExtentFlush { repair_id: ReconciliationId(580), extent_id: 155, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.488 DEBG 580 Flush extent 155 with f:2 g:2
Sep 22 23:22:25.488 DEBG Flush just extent 155 with f:2 and g:2
Sep 22 23:22:25.488 DEBG [1] It's time to notify for 580
Sep 22 23:22:25.488 INFO Completion from [1] id:580 status:true
Sep 22 23:22:25.488 INFO [581/752] Repair commands completed
Sep 22 23:22:25.488 INFO Pop front: ReconcileIO { id: ReconciliationId(581), op: ExtentClose { repair_id: ReconciliationId(581), extent_id: 155 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.488 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.488 INFO [0] received reconcile message
Sep 22 23:22:25.488 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(581), op: ExtentClose { repair_id: ReconciliationId(581), extent_id: 155 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.488 INFO [0] client ExtentClose { repair_id: ReconciliationId(581), extent_id: 155 }
Sep 22 23:22:25.488 INFO [1] received reconcile message
Sep 22 23:22:25.488 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(581), op: ExtentClose { repair_id: ReconciliationId(581), extent_id: 155 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.488 INFO [1] client ExtentClose { repair_id: ReconciliationId(581), extent_id: 155 }
Sep 22 23:22:25.488 INFO [2] received reconcile message
Sep 22 23:22:25.488 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(581), op: ExtentClose { repair_id: ReconciliationId(581), extent_id: 155 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.488 INFO [2] client ExtentClose { repair_id: ReconciliationId(581), extent_id: 155 }
Sep 22 23:22:25.489 DEBG 581 Close extent 155
Sep 22 23:22:25.489 DEBG 581 Close extent 155
Sep 22 23:22:25.489 DEBG 581 Close extent 155
Sep 22 23:22:25.490 DEBG [2] It's time to notify for 581
Sep 22 23:22:25.490 INFO Completion from [2] id:581 status:true
Sep 22 23:22:25.490 INFO [582/752] Repair commands completed
Sep 22 23:22:25.490 INFO Pop front: ReconcileIO { id: ReconciliationId(582), op: ExtentRepair { repair_id: ReconciliationId(582), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.490 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.490 INFO [0] received reconcile message
Sep 22 23:22:25.490 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(582), op: ExtentRepair { repair_id: ReconciliationId(582), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.490 INFO [0] client ExtentRepair { repair_id: ReconciliationId(582), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.490 INFO [0] Sending repair request ReconciliationId(582)
Sep 22 23:22:25.490 INFO [1] received reconcile message
Sep 22 23:22:25.490 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(582), op: ExtentRepair { repair_id: ReconciliationId(582), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.490 INFO [1] client ExtentRepair { repair_id: ReconciliationId(582), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.490 INFO [1] No action required ReconciliationId(582)
Sep 22 23:22:25.490 INFO [2] received reconcile message
Sep 22 23:22:25.490 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(582), op: ExtentRepair { repair_id: ReconciliationId(582), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.490 INFO [2] client ExtentRepair { repair_id: ReconciliationId(582), extent_id: 155, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.490 INFO [2] No action required ReconciliationId(582)
Sep 22 23:22:25.490 DEBG 582 Repair extent 155 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.490 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/09B.copy"
Sep 22 23:22:25.553 INFO accepted connection, remote_addr: 127.0.0.1:43168, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.553 TRCE incoming request, uri: /extent/155/files, method: GET, req_id: 3116d4a7-4c54-43f2-bb3d-3143c4975672, remote_addr: 127.0.0.1:43168, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.553 INFO request completed, latency_us: 281, response_code: 200, uri: /extent/155/files, method: GET, req_id: 3116d4a7-4c54-43f2-bb3d-3143c4975672, remote_addr: 127.0.0.1:43168, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.554 INFO eid:155 Found repair files: ["09B", "09B.db"]
Sep 22 23:22:25.554 TRCE incoming request, uri: /newextent/155/data, method: GET, req_id: 5657a3ec-3fc8-4c3f-b4a5-877673b97309, remote_addr: 127.0.0.1:43168, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.554 INFO request completed, latency_us: 376, response_code: 200, uri: /newextent/155/data, method: GET, req_id: 5657a3ec-3fc8-4c3f-b4a5-877673b97309, remote_addr: 127.0.0.1:43168, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.559 TRCE incoming request, uri: /newextent/155/db, method: GET, req_id: dd5a0f71-4f60-43d7-acb3-d05eabbf5b65, remote_addr: 127.0.0.1:43168, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.560 INFO request completed, latency_us: 309, response_code: 200, uri: /newextent/155/db, method: GET, req_id: dd5a0f71-4f60-43d7-acb3-d05eabbf5b65, remote_addr: 127.0.0.1:43168, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.561 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/09B.copy" to "/tmp/downstairs-zrMnlo6G/00/000/09B.replace"
Sep 22 23:22:25.561 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.562 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/09B.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.562 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09B"
Sep 22 23:22:25.562 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/09B.db"
Sep 22 23:22:25.563 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.563 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/09B.replace" to "/tmp/downstairs-zrMnlo6G/00/000/09B.completed"
Sep 22 23:22:25.563 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.563 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.563 DEBG [0] It's time to notify for 582
Sep 22 23:22:25.563 INFO Completion from [0] id:582 status:true
Sep 22 23:22:25.563 INFO [583/752] Repair commands completed
Sep 22 23:22:25.563 INFO Pop front: ReconcileIO { id: ReconciliationId(583), op: ExtentReopen { repair_id: ReconciliationId(583), extent_id: 155 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.563 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.563 INFO [0] received reconcile message
Sep 22 23:22:25.563 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(583), op: ExtentReopen { repair_id: ReconciliationId(583), extent_id: 155 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.563 INFO [0] client ExtentReopen { repair_id: ReconciliationId(583), extent_id: 155 }
Sep 22 23:22:25.563 INFO [1] received reconcile message
Sep 22 23:22:25.563 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(583), op: ExtentReopen { repair_id: ReconciliationId(583), extent_id: 155 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.563 INFO [1] client ExtentReopen { repair_id: ReconciliationId(583), extent_id: 155 }
Sep 22 23:22:25.563 INFO [2] received reconcile message
Sep 22 23:22:25.563 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(583), op: ExtentReopen { repair_id: ReconciliationId(583), extent_id: 155 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.563 INFO [2] client ExtentReopen { repair_id: ReconciliationId(583), extent_id: 155 }
Sep 22 23:22:25.564 DEBG 583 Reopen extent 155
Sep 22 23:22:25.564 DEBG 583 Reopen extent 155
Sep 22 23:22:25.565 DEBG 583 Reopen extent 155
Sep 22 23:22:25.565 DEBG [2] It's time to notify for 583
Sep 22 23:22:25.566 INFO Completion from [2] id:583 status:true
Sep 22 23:22:25.566 INFO [584/752] Repair commands completed
Sep 22 23:22:25.566 INFO Pop front: ReconcileIO { id: ReconciliationId(584), op: ExtentFlush { repair_id: ReconciliationId(584), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.566 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.566 INFO [0] received reconcile message
Sep 22 23:22:25.566 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(584), op: ExtentFlush { repair_id: ReconciliationId(584), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.566 INFO [0] client ExtentFlush { repair_id: ReconciliationId(584), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.566 INFO [1] received reconcile message
Sep 22 23:22:25.566 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(584), op: ExtentFlush { repair_id: ReconciliationId(584), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.566 INFO [1] client ExtentFlush { repair_id: ReconciliationId(584), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.566 INFO [2] received reconcile message
Sep 22 23:22:25.566 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(584), op: ExtentFlush { repair_id: ReconciliationId(584), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.566 INFO [2] client ExtentFlush { repair_id: ReconciliationId(584), extent_id: 0, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.566 DEBG 584 Flush extent 0 with f:2 g:2
Sep 22 23:22:25.566 DEBG Flush just extent 0 with f:2 and g:2
Sep 22 23:22:25.566 DEBG [1] It's time to notify for 584
Sep 22 23:22:25.566 INFO Completion from [1] id:584 status:true
Sep 22 23:22:25.566 INFO [585/752] Repair commands completed
Sep 22 23:22:25.566 INFO Pop front: ReconcileIO { id: ReconciliationId(585), op: ExtentClose { repair_id: ReconciliationId(585), extent_id: 0 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.566 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.566 INFO [0] received reconcile message
Sep 22 23:22:25.566 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(585), op: ExtentClose { repair_id: ReconciliationId(585), extent_id: 0 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.566 INFO [0] client ExtentClose { repair_id: ReconciliationId(585), extent_id: 0 }
Sep 22 23:22:25.566 INFO [1] received reconcile message
Sep 22 23:22:25.566 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(585), op: ExtentClose { repair_id: ReconciliationId(585), extent_id: 0 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.566 INFO [1] client ExtentClose { repair_id: ReconciliationId(585), extent_id: 0 }
Sep 22 23:22:25.566 INFO [2] received reconcile message
Sep 22 23:22:25.566 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(585), op: ExtentClose { repair_id: ReconciliationId(585), extent_id: 0 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.566 INFO [2] client ExtentClose { repair_id: ReconciliationId(585), extent_id: 0 }
Sep 22 23:22:25.567 DEBG 585 Close extent 0
Sep 22 23:22:25.567 DEBG 585 Close extent 0
Sep 22 23:22:25.567 DEBG 585 Close extent 0
Sep 22 23:22:25.568 DEBG [2] It's time to notify for 585
Sep 22 23:22:25.568 INFO Completion from [2] id:585 status:true
Sep 22 23:22:25.568 INFO [586/752] Repair commands completed
Sep 22 23:22:25.568 INFO Pop front: ReconcileIO { id: ReconciliationId(586), op: ExtentRepair { repair_id: ReconciliationId(586), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.568 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.568 INFO [0] received reconcile message
Sep 22 23:22:25.568 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(586), op: ExtentRepair { repair_id: ReconciliationId(586), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.568 INFO [0] client ExtentRepair { repair_id: ReconciliationId(586), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.568 INFO [0] Sending repair request ReconciliationId(586)
Sep 22 23:22:25.568 INFO [1] received reconcile message
Sep 22 23:22:25.568 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(586), op: ExtentRepair { repair_id: ReconciliationId(586), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.568 INFO [1] client ExtentRepair { repair_id: ReconciliationId(586), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.568 INFO [1] No action required ReconciliationId(586)
Sep 22 23:22:25.568 INFO [2] received reconcile message
Sep 22 23:22:25.568 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(586), op: ExtentRepair { repair_id: ReconciliationId(586), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.568 INFO [2] client ExtentRepair { repair_id: ReconciliationId(586), extent_id: 0, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.568 INFO [2] No action required ReconciliationId(586)
Sep 22 23:22:25.568 DEBG 586 Repair extent 0 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.568 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/000.copy"
Sep 22 23:22:25.621 DEBG up_ds_listen was notified
Sep 22 23:22:25.621 DEBG up_ds_listen process 1090
Sep 22 23:22:25.621 DEBG [A] ack job 1090:91, : downstairs
Sep 22 23:22:25.621 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:25.622 DEBG Flush     :1086 extent_limit None deps:[JobId(1085), JobId(1084)] res:true f:32 g:1
Sep 22 23:22:25.627 DEBG Read      :1087 deps:[JobId(1086)] res:true
Sep 22 23:22:25.632 INFO accepted connection, remote_addr: 127.0.0.1:49566, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.632 TRCE incoming request, uri: /extent/0/files, method: GET, req_id: 77e00068-f05b-45bb-b6c3-3b43650c38ea, remote_addr: 127.0.0.1:49566, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.632 INFO request completed, latency_us: 211, response_code: 200, uri: /extent/0/files, method: GET, req_id: 77e00068-f05b-45bb-b6c3-3b43650c38ea, remote_addr: 127.0.0.1:49566, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.633 INFO eid:0 Found repair files: ["000", "000.db"]
Sep 22 23:22:25.633 TRCE incoming request, uri: /newextent/0/data, method: GET, req_id: 71903321-8833-4637-b215-d43b720fd67d, remote_addr: 127.0.0.1:49566, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.633 INFO request completed, latency_us: 332, response_code: 200, uri: /newextent/0/data, method: GET, req_id: 71903321-8833-4637-b215-d43b720fd67d, remote_addr: 127.0.0.1:49566, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.638 TRCE incoming request, uri: /newextent/0/db, method: GET, req_id: fab1d54a-5b31-4e75-aa06-247bea7c7bb3, remote_addr: 127.0.0.1:49566, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.639 INFO request completed, latency_us: 318, response_code: 200, uri: /newextent/0/db, method: GET, req_id: fab1d54a-5b31-4e75-aa06-247bea7c7bb3, remote_addr: 127.0.0.1:49566, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.640 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/000.copy" to "/tmp/downstairs-zrMnlo6G/00/000/000.replace"
Sep 22 23:22:25.640 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.641 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/000.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.641 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/000"
Sep 22 23:22:25.641 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/000.db"
Sep 22 23:22:25.641 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.641 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/000.replace" to "/tmp/downstairs-zrMnlo6G/00/000/000.completed"
Sep 22 23:22:25.641 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.641 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.641 DEBG [0] It's time to notify for 586
Sep 22 23:22:25.642 INFO Completion from [0] id:586 status:true
Sep 22 23:22:25.642 INFO [587/752] Repair commands completed
Sep 22 23:22:25.642 INFO Pop front: ReconcileIO { id: ReconciliationId(587), op: ExtentReopen { repair_id: ReconciliationId(587), extent_id: 0 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.642 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.642 INFO [0] received reconcile message
Sep 22 23:22:25.642 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(587), op: ExtentReopen { repair_id: ReconciliationId(587), extent_id: 0 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.642 INFO [0] client ExtentReopen { repair_id: ReconciliationId(587), extent_id: 0 }
Sep 22 23:22:25.642 INFO [1] received reconcile message
Sep 22 23:22:25.642 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(587), op: ExtentReopen { repair_id: ReconciliationId(587), extent_id: 0 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.642 INFO [1] client ExtentReopen { repair_id: ReconciliationId(587), extent_id: 0 }
Sep 22 23:22:25.642 INFO [2] received reconcile message
Sep 22 23:22:25.642 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(587), op: ExtentReopen { repair_id: ReconciliationId(587), extent_id: 0 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.642 INFO [2] client ExtentReopen { repair_id: ReconciliationId(587), extent_id: 0 }
Sep 22 23:22:25.642 DEBG 587 Reopen extent 0
Sep 22 23:22:25.643 DEBG 587 Reopen extent 0
Sep 22 23:22:25.643 DEBG 587 Reopen extent 0
Sep 22 23:22:25.644 DEBG [2] It's time to notify for 587
Sep 22 23:22:25.644 INFO Completion from [2] id:587 status:true
Sep 22 23:22:25.644 INFO [588/752] Repair commands completed
Sep 22 23:22:25.644 INFO Pop front: ReconcileIO { id: ReconciliationId(588), op: ExtentFlush { repair_id: ReconciliationId(588), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.644 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.644 INFO [0] received reconcile message
Sep 22 23:22:25.644 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(588), op: ExtentFlush { repair_id: ReconciliationId(588), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.644 INFO [0] client ExtentFlush { repair_id: ReconciliationId(588), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.644 INFO [1] received reconcile message
Sep 22 23:22:25.644 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(588), op: ExtentFlush { repair_id: ReconciliationId(588), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.644 INFO [1] client ExtentFlush { repair_id: ReconciliationId(588), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.644 INFO [2] received reconcile message
Sep 22 23:22:25.644 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(588), op: ExtentFlush { repair_id: ReconciliationId(588), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.644 INFO [2] client ExtentFlush { repair_id: ReconciliationId(588), extent_id: 164, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.644 DEBG 588 Flush extent 164 with f:2 g:2
Sep 22 23:22:25.644 DEBG Flush just extent 164 with f:2 and g:2
Sep 22 23:22:25.644 DEBG [1] It's time to notify for 588
Sep 22 23:22:25.644 INFO Completion from [1] id:588 status:true
Sep 22 23:22:25.644 INFO [589/752] Repair commands completed
Sep 22 23:22:25.644 INFO Pop front: ReconcileIO { id: ReconciliationId(589), op: ExtentClose { repair_id: ReconciliationId(589), extent_id: 164 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.644 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.644 INFO [0] received reconcile message
Sep 22 23:22:25.644 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(589), op: ExtentClose { repair_id: ReconciliationId(589), extent_id: 164 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.644 INFO [0] client ExtentClose { repair_id: ReconciliationId(589), extent_id: 164 }
Sep 22 23:22:25.645 INFO [1] received reconcile message
Sep 22 23:22:25.645 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(589), op: ExtentClose { repair_id: ReconciliationId(589), extent_id: 164 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.645 INFO [1] client ExtentClose { repair_id: ReconciliationId(589), extent_id: 164 }
Sep 22 23:22:25.645 INFO [2] received reconcile message
Sep 22 23:22:25.645 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(589), op: ExtentClose { repair_id: ReconciliationId(589), extent_id: 164 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.645 INFO [2] client ExtentClose { repair_id: ReconciliationId(589), extent_id: 164 }
Sep 22 23:22:25.645 DEBG 589 Close extent 164
Sep 22 23:22:25.645 DEBG 589 Close extent 164
Sep 22 23:22:25.645 DEBG 589 Close extent 164
Sep 22 23:22:25.646 DEBG [2] It's time to notify for 589
Sep 22 23:22:25.646 INFO Completion from [2] id:589 status:true
Sep 22 23:22:25.646 INFO [590/752] Repair commands completed
Sep 22 23:22:25.646 INFO Pop front: ReconcileIO { id: ReconciliationId(590), op: ExtentRepair { repair_id: ReconciliationId(590), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.646 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.646 INFO [0] received reconcile message
Sep 22 23:22:25.646 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(590), op: ExtentRepair { repair_id: ReconciliationId(590), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.646 INFO [0] client ExtentRepair { repair_id: ReconciliationId(590), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.646 INFO [0] Sending repair request ReconciliationId(590)
Sep 22 23:22:25.646 INFO [1] received reconcile message
Sep 22 23:22:25.646 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(590), op: ExtentRepair { repair_id: ReconciliationId(590), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.646 INFO [1] client ExtentRepair { repair_id: ReconciliationId(590), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.646 INFO [1] No action required ReconciliationId(590)
Sep 22 23:22:25.646 INFO [2] received reconcile message
Sep 22 23:22:25.646 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(590), op: ExtentRepair { repair_id: ReconciliationId(590), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.646 INFO [2] client ExtentRepair { repair_id: ReconciliationId(590), extent_id: 164, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.646 INFO [2] No action required ReconciliationId(590)
Sep 22 23:22:25.646 DEBG 590 Repair extent 164 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.646 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0A4.copy"
Sep 22 23:22:25.649 DEBG [rc] retire 1086 clears [JobId(1085), JobId(1086)], : downstairs
Sep 22 23:22:25.652 INFO [lossy] sleeping 1 second
Sep 22 23:22:25.712 INFO accepted connection, remote_addr: 127.0.0.1:64591, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.712 TRCE incoming request, uri: /extent/164/files, method: GET, req_id: 1385ac13-c900-4093-a2ec-51bea160704d, remote_addr: 127.0.0.1:64591, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.712 INFO request completed, latency_us: 210, response_code: 200, uri: /extent/164/files, method: GET, req_id: 1385ac13-c900-4093-a2ec-51bea160704d, remote_addr: 127.0.0.1:64591, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.713 INFO eid:164 Found repair files: ["0A4", "0A4.db"]
Sep 22 23:22:25.713 TRCE incoming request, uri: /newextent/164/data, method: GET, req_id: 9e2524b9-95d3-48cd-aa53-0a4e4fb2f974, remote_addr: 127.0.0.1:64591, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.713 INFO request completed, latency_us: 327, response_code: 200, uri: /newextent/164/data, method: GET, req_id: 9e2524b9-95d3-48cd-aa53-0a4e4fb2f974, remote_addr: 127.0.0.1:64591, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.718 TRCE incoming request, uri: /newextent/164/db, method: GET, req_id: 80cb5eb5-5e70-40ac-9638-f5b782327418, remote_addr: 127.0.0.1:64591, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.718 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/164/db, method: GET, req_id: 80cb5eb5-5e70-40ac-9638-f5b782327418, remote_addr: 127.0.0.1:64591, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.720 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0A4.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0A4.replace"
Sep 22 23:22:25.720 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.720 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0A4.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.721 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A4"
Sep 22 23:22:25.721 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0A4.db"
Sep 22 23:22:25.721 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.721 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0A4.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0A4.completed"
Sep 22 23:22:25.721 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.721 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.721 DEBG [0] It's time to notify for 590
Sep 22 23:22:25.721 INFO Completion from [0] id:590 status:true
Sep 22 23:22:25.721 INFO [591/752] Repair commands completed
Sep 22 23:22:25.721 INFO Pop front: ReconcileIO { id: ReconciliationId(591), op: ExtentReopen { repair_id: ReconciliationId(591), extent_id: 164 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.721 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.721 INFO [0] received reconcile message
Sep 22 23:22:25.721 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(591), op: ExtentReopen { repair_id: ReconciliationId(591), extent_id: 164 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.721 INFO [0] client ExtentReopen { repair_id: ReconciliationId(591), extent_id: 164 }
Sep 22 23:22:25.721 INFO [1] received reconcile message
Sep 22 23:22:25.721 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(591), op: ExtentReopen { repair_id: ReconciliationId(591), extent_id: 164 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.721 INFO [1] client ExtentReopen { repair_id: ReconciliationId(591), extent_id: 164 }
Sep 22 23:22:25.721 INFO [2] received reconcile message
Sep 22 23:22:25.722 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(591), op: ExtentReopen { repair_id: ReconciliationId(591), extent_id: 164 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.722 INFO [2] client ExtentReopen { repair_id: ReconciliationId(591), extent_id: 164 }
Sep 22 23:22:25.722 DEBG 591 Reopen extent 164
Sep 22 23:22:25.722 DEBG 591 Reopen extent 164
Sep 22 23:22:25.723 DEBG 591 Reopen extent 164
Sep 22 23:22:25.723 DEBG [2] It's time to notify for 591
Sep 22 23:22:25.723 INFO Completion from [2] id:591 status:true
Sep 22 23:22:25.723 INFO [592/752] Repair commands completed
Sep 22 23:22:25.723 INFO Pop front: ReconcileIO { id: ReconciliationId(592), op: ExtentFlush { repair_id: ReconciliationId(592), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.723 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.724 INFO [0] received reconcile message
Sep 22 23:22:25.724 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(592), op: ExtentFlush { repair_id: ReconciliationId(592), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.724 INFO [0] client ExtentFlush { repair_id: ReconciliationId(592), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.724 INFO [1] received reconcile message
Sep 22 23:22:25.724 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(592), op: ExtentFlush { repair_id: ReconciliationId(592), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.724 INFO [1] client ExtentFlush { repair_id: ReconciliationId(592), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.724 INFO [2] received reconcile message
Sep 22 23:22:25.724 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(592), op: ExtentFlush { repair_id: ReconciliationId(592), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.724 INFO [2] client ExtentFlush { repair_id: ReconciliationId(592), extent_id: 24, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.724 DEBG 592 Flush extent 24 with f:2 g:2
Sep 22 23:22:25.724 DEBG Flush just extent 24 with f:2 and g:2
Sep 22 23:22:25.724 DEBG [1] It's time to notify for 592
Sep 22 23:22:25.724 INFO Completion from [1] id:592 status:true
Sep 22 23:22:25.724 INFO [593/752] Repair commands completed
Sep 22 23:22:25.724 INFO Pop front: ReconcileIO { id: ReconciliationId(593), op: ExtentClose { repair_id: ReconciliationId(593), extent_id: 24 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.724 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.724 INFO [0] received reconcile message
Sep 22 23:22:25.724 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(593), op: ExtentClose { repair_id: ReconciliationId(593), extent_id: 24 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.724 INFO [0] client ExtentClose { repair_id: ReconciliationId(593), extent_id: 24 }
Sep 22 23:22:25.724 INFO [1] received reconcile message
Sep 22 23:22:25.724 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(593), op: ExtentClose { repair_id: ReconciliationId(593), extent_id: 24 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.724 INFO [1] client ExtentClose { repair_id: ReconciliationId(593), extent_id: 24 }
Sep 22 23:22:25.724 INFO [2] received reconcile message
Sep 22 23:22:25.724 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(593), op: ExtentClose { repair_id: ReconciliationId(593), extent_id: 24 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.724 INFO [2] client ExtentClose { repair_id: ReconciliationId(593), extent_id: 24 }
Sep 22 23:22:25.724 DEBG 593 Close extent 24
Sep 22 23:22:25.725 DEBG 593 Close extent 24
Sep 22 23:22:25.725 DEBG 593 Close extent 24
Sep 22 23:22:25.725 DEBG [2] It's time to notify for 593
Sep 22 23:22:25.725 INFO Completion from [2] id:593 status:true
Sep 22 23:22:25.725 INFO [594/752] Repair commands completed
Sep 22 23:22:25.725 INFO Pop front: ReconcileIO { id: ReconciliationId(594), op: ExtentRepair { repair_id: ReconciliationId(594), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.725 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.726 INFO [0] received reconcile message
Sep 22 23:22:25.726 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(594), op: ExtentRepair { repair_id: ReconciliationId(594), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.726 INFO [0] client ExtentRepair { repair_id: ReconciliationId(594), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.726 INFO [0] Sending repair request ReconciliationId(594)
Sep 22 23:22:25.726 INFO [1] received reconcile message
Sep 22 23:22:25.726 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(594), op: ExtentRepair { repair_id: ReconciliationId(594), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.726 INFO [1] client ExtentRepair { repair_id: ReconciliationId(594), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.726 INFO [1] No action required ReconciliationId(594)
Sep 22 23:22:25.726 INFO [2] received reconcile message
Sep 22 23:22:25.726 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(594), op: ExtentRepair { repair_id: ReconciliationId(594), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.726 INFO [2] client ExtentRepair { repair_id: ReconciliationId(594), extent_id: 24, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.726 INFO [2] No action required ReconciliationId(594)
Sep 22 23:22:25.726 DEBG 594 Repair extent 24 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.726 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/018.copy"
Sep 22 23:22:25.790 INFO accepted connection, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.791 TRCE incoming request, uri: /extent/24/files, method: GET, req_id: e9a2a2d9-0228-479d-a1a1-69fbc26cc87d, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.791 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/24/files, method: GET, req_id: e9a2a2d9-0228-479d-a1a1-69fbc26cc87d, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.791 INFO eid:24 Found repair files: ["018", "018.db"]
Sep 22 23:22:25.791 TRCE incoming request, uri: /newextent/24/data, method: GET, req_id: 7e565e8c-534b-4d1d-ab37-22d39760078c, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.792 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/24/data, method: GET, req_id: 7e565e8c-534b-4d1d-ab37-22d39760078c, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.796 TRCE incoming request, uri: /newextent/24/db, method: GET, req_id: b096e23f-727c-4d77-a7cc-7b67c7ddc61a, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.797 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/24/db, method: GET, req_id: b096e23f-727c-4d77-a7cc-7b67c7ddc61a, remote_addr: 127.0.0.1:60516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.798 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/018.copy" to "/tmp/downstairs-zrMnlo6G/00/000/018.replace"
Sep 22 23:22:25.798 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.799 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/018.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.799 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/018"
Sep 22 23:22:25.799 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/018.db"
Sep 22 23:22:25.799 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.799 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/018.replace" to "/tmp/downstairs-zrMnlo6G/00/000/018.completed"
Sep 22 23:22:25.799 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.799 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.799 DEBG [0] It's time to notify for 594
Sep 22 23:22:25.799 INFO Completion from [0] id:594 status:true
Sep 22 23:22:25.799 INFO [595/752] Repair commands completed
Sep 22 23:22:25.799 INFO Pop front: ReconcileIO { id: ReconciliationId(595), op: ExtentReopen { repair_id: ReconciliationId(595), extent_id: 24 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.799 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.800 INFO [0] received reconcile message
Sep 22 23:22:25.800 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(595), op: ExtentReopen { repair_id: ReconciliationId(595), extent_id: 24 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.800 INFO [0] client ExtentReopen { repair_id: ReconciliationId(595), extent_id: 24 }
Sep 22 23:22:25.800 INFO [1] received reconcile message
Sep 22 23:22:25.800 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(595), op: ExtentReopen { repair_id: ReconciliationId(595), extent_id: 24 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.800 INFO [1] client ExtentReopen { repair_id: ReconciliationId(595), extent_id: 24 }
Sep 22 23:22:25.800 INFO [2] received reconcile message
Sep 22 23:22:25.800 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(595), op: ExtentReopen { repair_id: ReconciliationId(595), extent_id: 24 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.800 INFO [2] client ExtentReopen { repair_id: ReconciliationId(595), extent_id: 24 }
Sep 22 23:22:25.800 DEBG 595 Reopen extent 24
Sep 22 23:22:25.800 DEBG 595 Reopen extent 24
Sep 22 23:22:25.801 DEBG 595 Reopen extent 24
Sep 22 23:22:25.802 DEBG [2] It's time to notify for 595
Sep 22 23:22:25.802 INFO Completion from [2] id:595 status:true
Sep 22 23:22:25.802 INFO [596/752] Repair commands completed
Sep 22 23:22:25.802 INFO Pop front: ReconcileIO { id: ReconciliationId(596), op: ExtentFlush { repair_id: ReconciliationId(596), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.802 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.802 INFO [0] received reconcile message
Sep 22 23:22:25.802 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(596), op: ExtentFlush { repair_id: ReconciliationId(596), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.802 INFO [0] client ExtentFlush { repair_id: ReconciliationId(596), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.802 INFO [1] received reconcile message
Sep 22 23:22:25.802 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(596), op: ExtentFlush { repair_id: ReconciliationId(596), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.802 INFO [1] client ExtentFlush { repair_id: ReconciliationId(596), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.802 INFO [2] received reconcile message
Sep 22 23:22:25.802 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(596), op: ExtentFlush { repair_id: ReconciliationId(596), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.802 INFO [2] client ExtentFlush { repair_id: ReconciliationId(596), extent_id: 39, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.802 DEBG 596 Flush extent 39 with f:2 g:2
Sep 22 23:22:25.802 DEBG Flush just extent 39 with f:2 and g:2
Sep 22 23:22:25.802 DEBG [1] It's time to notify for 596
Sep 22 23:22:25.802 INFO Completion from [1] id:596 status:true
Sep 22 23:22:25.802 INFO [597/752] Repair commands completed
Sep 22 23:22:25.802 INFO Pop front: ReconcileIO { id: ReconciliationId(597), op: ExtentClose { repair_id: ReconciliationId(597), extent_id: 39 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.802 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.802 INFO [0] received reconcile message
Sep 22 23:22:25.802 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(597), op: ExtentClose { repair_id: ReconciliationId(597), extent_id: 39 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.802 INFO [0] client ExtentClose { repair_id: ReconciliationId(597), extent_id: 39 }
Sep 22 23:22:25.802 INFO [1] received reconcile message
Sep 22 23:22:25.802 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(597), op: ExtentClose { repair_id: ReconciliationId(597), extent_id: 39 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.802 INFO [1] client ExtentClose { repair_id: ReconciliationId(597), extent_id: 39 }
Sep 22 23:22:25.802 INFO [2] received reconcile message
Sep 22 23:22:25.802 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(597), op: ExtentClose { repair_id: ReconciliationId(597), extent_id: 39 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.802 INFO [2] client ExtentClose { repair_id: ReconciliationId(597), extent_id: 39 }
Sep 22 23:22:25.803 DEBG 597 Close extent 39
Sep 22 23:22:25.803 DEBG 597 Close extent 39
Sep 22 23:22:25.803 DEBG 597 Close extent 39
Sep 22 23:22:25.804 DEBG [2] It's time to notify for 597
Sep 22 23:22:25.804 INFO Completion from [2] id:597 status:true
Sep 22 23:22:25.804 INFO [598/752] Repair commands completed
Sep 22 23:22:25.804 INFO Pop front: ReconcileIO { id: ReconciliationId(598), op: ExtentRepair { repair_id: ReconciliationId(598), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.804 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.804 INFO [0] received reconcile message
Sep 22 23:22:25.804 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(598), op: ExtentRepair { repair_id: ReconciliationId(598), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.804 INFO [0] client ExtentRepair { repair_id: ReconciliationId(598), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.804 INFO [0] Sending repair request ReconciliationId(598)
Sep 22 23:22:25.804 INFO [1] received reconcile message
Sep 22 23:22:25.804 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(598), op: ExtentRepair { repair_id: ReconciliationId(598), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.804 INFO [1] client ExtentRepair { repair_id: ReconciliationId(598), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.804 INFO [1] No action required ReconciliationId(598)
Sep 22 23:22:25.804 INFO [2] received reconcile message
Sep 22 23:22:25.804 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(598), op: ExtentRepair { repair_id: ReconciliationId(598), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.804 INFO [2] client ExtentRepair { repair_id: ReconciliationId(598), extent_id: 39, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.804 INFO [2] No action required ReconciliationId(598)
Sep 22 23:22:25.804 DEBG 598 Repair extent 39 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.804 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/027.copy"
Sep 22 23:22:25.869 INFO accepted connection, remote_addr: 127.0.0.1:50542, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.869 TRCE incoming request, uri: /extent/39/files, method: GET, req_id: b8a74093-007e-4eb5-9e9f-66d132ff9366, remote_addr: 127.0.0.1:50542, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.869 INFO request completed, latency_us: 203, response_code: 200, uri: /extent/39/files, method: GET, req_id: b8a74093-007e-4eb5-9e9f-66d132ff9366, remote_addr: 127.0.0.1:50542, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.870 INFO eid:39 Found repair files: ["027", "027.db"]
Sep 22 23:22:25.870 TRCE incoming request, uri: /newextent/39/data, method: GET, req_id: 407df6bc-f6d0-4f07-805a-0c90f1c13280, remote_addr: 127.0.0.1:50542, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.870 INFO request completed, latency_us: 327, response_code: 200, uri: /newextent/39/data, method: GET, req_id: 407df6bc-f6d0-4f07-805a-0c90f1c13280, remote_addr: 127.0.0.1:50542, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.875 TRCE incoming request, uri: /newextent/39/db, method: GET, req_id: a3e140db-4494-4a2c-b644-3e5892e628c3, remote_addr: 127.0.0.1:50542, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.875 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/39/db, method: GET, req_id: a3e140db-4494-4a2c-b644-3e5892e628c3, remote_addr: 127.0.0.1:50542, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.876 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/027.copy" to "/tmp/downstairs-zrMnlo6G/00/000/027.replace"
Sep 22 23:22:25.876 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.877 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/027.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.877 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/027"
Sep 22 23:22:25.877 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/027.db"
Sep 22 23:22:25.877 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.877 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/027.replace" to "/tmp/downstairs-zrMnlo6G/00/000/027.completed"
Sep 22 23:22:25.877 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.878 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.878 DEBG [0] It's time to notify for 598
Sep 22 23:22:25.878 INFO Completion from [0] id:598 status:true
Sep 22 23:22:25.878 INFO [599/752] Repair commands completed
Sep 22 23:22:25.878 INFO Pop front: ReconcileIO { id: ReconciliationId(599), op: ExtentReopen { repair_id: ReconciliationId(599), extent_id: 39 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.878 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.878 INFO [0] received reconcile message
Sep 22 23:22:25.878 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(599), op: ExtentReopen { repair_id: ReconciliationId(599), extent_id: 39 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.878 INFO [0] client ExtentReopen { repair_id: ReconciliationId(599), extent_id: 39 }
Sep 22 23:22:25.878 INFO [1] received reconcile message
Sep 22 23:22:25.878 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(599), op: ExtentReopen { repair_id: ReconciliationId(599), extent_id: 39 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.878 INFO [1] client ExtentReopen { repair_id: ReconciliationId(599), extent_id: 39 }
Sep 22 23:22:25.878 INFO [2] received reconcile message
Sep 22 23:22:25.878 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(599), op: ExtentReopen { repair_id: ReconciliationId(599), extent_id: 39 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.878 INFO [2] client ExtentReopen { repair_id: ReconciliationId(599), extent_id: 39 }
Sep 22 23:22:25.878 DEBG 599 Reopen extent 39
Sep 22 23:22:25.879 DEBG 599 Reopen extent 39
Sep 22 23:22:25.879 DEBG 599 Reopen extent 39
Sep 22 23:22:25.880 DEBG [2] It's time to notify for 599
Sep 22 23:22:25.880 INFO Completion from [2] id:599 status:true
Sep 22 23:22:25.880 INFO [600/752] Repair commands completed
Sep 22 23:22:25.880 INFO Pop front: ReconcileIO { id: ReconciliationId(600), op: ExtentFlush { repair_id: ReconciliationId(600), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.880 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.880 INFO [0] received reconcile message
Sep 22 23:22:25.880 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(600), op: ExtentFlush { repair_id: ReconciliationId(600), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.880 INFO [0] client ExtentFlush { repair_id: ReconciliationId(600), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.880 INFO [1] received reconcile message
Sep 22 23:22:25.880 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(600), op: ExtentFlush { repair_id: ReconciliationId(600), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.880 INFO [1] client ExtentFlush { repair_id: ReconciliationId(600), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.880 INFO [2] received reconcile message
Sep 22 23:22:25.880 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(600), op: ExtentFlush { repair_id: ReconciliationId(600), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.880 INFO [2] client ExtentFlush { repair_id: ReconciliationId(600), extent_id: 48, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.880 DEBG 600 Flush extent 48 with f:2 g:2
Sep 22 23:22:25.880 DEBG Flush just extent 48 with f:2 and g:2
Sep 22 23:22:25.881 DEBG [1] It's time to notify for 600
Sep 22 23:22:25.881 INFO Completion from [1] id:600 status:true
Sep 22 23:22:25.881 INFO [601/752] Repair commands completed
Sep 22 23:22:25.881 INFO Pop front: ReconcileIO { id: ReconciliationId(601), op: ExtentClose { repair_id: ReconciliationId(601), extent_id: 48 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.881 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.881 INFO [0] received reconcile message
Sep 22 23:22:25.881 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(601), op: ExtentClose { repair_id: ReconciliationId(601), extent_id: 48 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.881 INFO [0] client ExtentClose { repair_id: ReconciliationId(601), extent_id: 48 }
Sep 22 23:22:25.881 INFO [1] received reconcile message
Sep 22 23:22:25.881 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(601), op: ExtentClose { repair_id: ReconciliationId(601), extent_id: 48 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.881 INFO [1] client ExtentClose { repair_id: ReconciliationId(601), extent_id: 48 }
Sep 22 23:22:25.881 INFO [2] received reconcile message
Sep 22 23:22:25.881 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(601), op: ExtentClose { repair_id: ReconciliationId(601), extent_id: 48 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.881 INFO [2] client ExtentClose { repair_id: ReconciliationId(601), extent_id: 48 }
Sep 22 23:22:25.881 DEBG 601 Close extent 48
Sep 22 23:22:25.881 DEBG 601 Close extent 48
Sep 22 23:22:25.882 DEBG 601 Close extent 48
Sep 22 23:22:25.882 DEBG [2] It's time to notify for 601
Sep 22 23:22:25.882 INFO Completion from [2] id:601 status:true
Sep 22 23:22:25.882 INFO [602/752] Repair commands completed
Sep 22 23:22:25.882 INFO Pop front: ReconcileIO { id: ReconciliationId(602), op: ExtentRepair { repair_id: ReconciliationId(602), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.882 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.882 INFO [0] received reconcile message
Sep 22 23:22:25.882 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(602), op: ExtentRepair { repair_id: ReconciliationId(602), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.882 INFO [0] client ExtentRepair { repair_id: ReconciliationId(602), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.882 INFO [0] Sending repair request ReconciliationId(602)
Sep 22 23:22:25.882 INFO [1] received reconcile message
Sep 22 23:22:25.882 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(602), op: ExtentRepair { repair_id: ReconciliationId(602), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.882 INFO [1] client ExtentRepair { repair_id: ReconciliationId(602), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.882 INFO [1] No action required ReconciliationId(602)
Sep 22 23:22:25.882 INFO [2] received reconcile message
Sep 22 23:22:25.882 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(602), op: ExtentRepair { repair_id: ReconciliationId(602), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.882 INFO [2] client ExtentRepair { repair_id: ReconciliationId(602), extent_id: 48, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.882 INFO [2] No action required ReconciliationId(602)
Sep 22 23:22:25.883 DEBG 602 Repair extent 48 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.883 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/030.copy"
Sep 22 23:22:25.946 INFO accepted connection, remote_addr: 127.0.0.1:41819, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.946 TRCE incoming request, uri: /extent/48/files, method: GET, req_id: 5e3137d7-a847-43fc-bc05-a938a8c698e3, remote_addr: 127.0.0.1:41819, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.946 INFO request completed, latency_us: 196, response_code: 200, uri: /extent/48/files, method: GET, req_id: 5e3137d7-a847-43fc-bc05-a938a8c698e3, remote_addr: 127.0.0.1:41819, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.946 INFO eid:48 Found repair files: ["030", "030.db"]
Sep 22 23:22:25.947 TRCE incoming request, uri: /newextent/48/data, method: GET, req_id: 2fc3a3f7-c9bd-4aaa-9642-3d90b4e5c783, remote_addr: 127.0.0.1:41819, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.947 INFO request completed, latency_us: 257, response_code: 200, uri: /newextent/48/data, method: GET, req_id: 2fc3a3f7-c9bd-4aaa-9642-3d90b4e5c783, remote_addr: 127.0.0.1:41819, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.952 TRCE incoming request, uri: /newextent/48/db, method: GET, req_id: d5972454-a3b3-45c4-b299-a8c0a6668dea, remote_addr: 127.0.0.1:41819, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.952 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/48/db, method: GET, req_id: d5972454-a3b3-45c4-b299-a8c0a6668dea, remote_addr: 127.0.0.1:41819, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:25.953 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/030.copy" to "/tmp/downstairs-zrMnlo6G/00/000/030.replace"
Sep 22 23:22:25.953 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.954 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/030.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.954 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/030"
Sep 22 23:22:25.954 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/030.db"
Sep 22 23:22:25.954 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.954 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/030.replace" to "/tmp/downstairs-zrMnlo6G/00/000/030.completed"
Sep 22 23:22:25.954 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.954 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:25.954 DEBG [0] It's time to notify for 602
Sep 22 23:22:25.955 INFO Completion from [0] id:602 status:true
Sep 22 23:22:25.955 INFO [603/752] Repair commands completed
Sep 22 23:22:25.955 INFO Pop front: ReconcileIO { id: ReconciliationId(603), op: ExtentReopen { repair_id: ReconciliationId(603), extent_id: 48 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.955 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.955 INFO [0] received reconcile message
Sep 22 23:22:25.955 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(603), op: ExtentReopen { repair_id: ReconciliationId(603), extent_id: 48 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.955 INFO [0] client ExtentReopen { repair_id: ReconciliationId(603), extent_id: 48 }
Sep 22 23:22:25.955 INFO [1] received reconcile message
Sep 22 23:22:25.955 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(603), op: ExtentReopen { repair_id: ReconciliationId(603), extent_id: 48 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.955 INFO [1] client ExtentReopen { repair_id: ReconciliationId(603), extent_id: 48 }
Sep 22 23:22:25.955 INFO [2] received reconcile message
Sep 22 23:22:25.955 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(603), op: ExtentReopen { repair_id: ReconciliationId(603), extent_id: 48 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.955 INFO [2] client ExtentReopen { repair_id: ReconciliationId(603), extent_id: 48 }
Sep 22 23:22:25.955 DEBG 603 Reopen extent 48
Sep 22 23:22:25.956 DEBG 603 Reopen extent 48
Sep 22 23:22:25.956 DEBG 603 Reopen extent 48
Sep 22 23:22:25.957 DEBG [2] It's time to notify for 603
Sep 22 23:22:25.957 INFO Completion from [2] id:603 status:true
Sep 22 23:22:25.957 INFO [604/752] Repair commands completed
Sep 22 23:22:25.957 INFO Pop front: ReconcileIO { id: ReconciliationId(604), op: ExtentFlush { repair_id: ReconciliationId(604), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.957 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.957 INFO [0] received reconcile message
Sep 22 23:22:25.957 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(604), op: ExtentFlush { repair_id: ReconciliationId(604), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.957 INFO [0] client ExtentFlush { repair_id: ReconciliationId(604), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.957 INFO [1] received reconcile message
Sep 22 23:22:25.957 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(604), op: ExtentFlush { repair_id: ReconciliationId(604), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:25.957 INFO [1] client ExtentFlush { repair_id: ReconciliationId(604), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.957 INFO [2] received reconcile message
Sep 22 23:22:25.957 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(604), op: ExtentFlush { repair_id: ReconciliationId(604), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.957 INFO [2] client ExtentFlush { repair_id: ReconciliationId(604), extent_id: 16, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:25.957 DEBG 604 Flush extent 16 with f:2 g:2
Sep 22 23:22:25.957 DEBG Flush just extent 16 with f:2 and g:2
Sep 22 23:22:25.957 DEBG [1] It's time to notify for 604
Sep 22 23:22:25.957 INFO Completion from [1] id:604 status:true
Sep 22 23:22:25.957 INFO [605/752] Repair commands completed
Sep 22 23:22:25.957 INFO Pop front: ReconcileIO { id: ReconciliationId(605), op: ExtentClose { repair_id: ReconciliationId(605), extent_id: 16 }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.957 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.957 INFO [0] received reconcile message
Sep 22 23:22:25.957 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(605), op: ExtentClose { repair_id: ReconciliationId(605), extent_id: 16 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.957 INFO [0] client ExtentClose { repair_id: ReconciliationId(605), extent_id: 16 }
Sep 22 23:22:25.957 INFO [1] received reconcile message
Sep 22 23:22:25.958 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(605), op: ExtentClose { repair_id: ReconciliationId(605), extent_id: 16 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.958 INFO [1] client ExtentClose { repair_id: ReconciliationId(605), extent_id: 16 }
Sep 22 23:22:25.958 INFO [2] received reconcile message
Sep 22 23:22:25.958 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(605), op: ExtentClose { repair_id: ReconciliationId(605), extent_id: 16 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:25.958 INFO [2] client ExtentClose { repair_id: ReconciliationId(605), extent_id: 16 }
Sep 22 23:22:25.958 DEBG 605 Close extent 16
Sep 22 23:22:25.958 DEBG 605 Close extent 16
Sep 22 23:22:25.958 DEBG 605 Close extent 16
Sep 22 23:22:25.959 DEBG [2] It's time to notify for 605
Sep 22 23:22:25.959 INFO Completion from [2] id:605 status:true
Sep 22 23:22:25.959 INFO [606/752] Repair commands completed
Sep 22 23:22:25.959 INFO Pop front: ReconcileIO { id: ReconciliationId(606), op: ExtentRepair { repair_id: ReconciliationId(606), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:25.959 INFO Sent repair work, now wait for resp
Sep 22 23:22:25.959 INFO [0] received reconcile message
Sep 22 23:22:25.959 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(606), op: ExtentRepair { repair_id: ReconciliationId(606), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:25.959 INFO [0] client ExtentRepair { repair_id: ReconciliationId(606), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.959 INFO [0] Sending repair request ReconciliationId(606)
Sep 22 23:22:25.959 INFO [1] received reconcile message
Sep 22 23:22:25.959 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(606), op: ExtentRepair { repair_id: ReconciliationId(606), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:25.959 INFO [1] client ExtentRepair { repair_id: ReconciliationId(606), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.959 INFO [1] No action required ReconciliationId(606)
Sep 22 23:22:25.959 INFO [2] received reconcile message
Sep 22 23:22:25.959 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(606), op: ExtentRepair { repair_id: ReconciliationId(606), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:25.959 INFO [2] client ExtentRepair { repair_id: ReconciliationId(606), extent_id: 16, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:25.959 INFO [2] No action required ReconciliationId(606)
Sep 22 23:22:25.959 DEBG 606 Repair extent 16 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:25.959 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/010.copy"
Sep 22 23:22:26.025 INFO accepted connection, remote_addr: 127.0.0.1:50664, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.025 TRCE incoming request, uri: /extent/16/files, method: GET, req_id: dd9029d7-5cf8-47a7-81f0-72cb97a746a5, remote_addr: 127.0.0.1:50664, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.026 INFO request completed, latency_us: 245, response_code: 200, uri: /extent/16/files, method: GET, req_id: dd9029d7-5cf8-47a7-81f0-72cb97a746a5, remote_addr: 127.0.0.1:50664, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.026 INFO eid:16 Found repair files: ["010", "010.db"]
Sep 22 23:22:26.026 TRCE incoming request, uri: /newextent/16/data, method: GET, req_id: e16b43fc-6511-44f8-9f3c-dca5d3b41908, remote_addr: 127.0.0.1:50664, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.027 INFO request completed, latency_us: 336, response_code: 200, uri: /newextent/16/data, method: GET, req_id: e16b43fc-6511-44f8-9f3c-dca5d3b41908, remote_addr: 127.0.0.1:50664, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.029 DEBG [0] Read AckReady 1091, : downstairs
Sep 22 23:22:26.030 DEBG up_ds_listen was notified
Sep 22 23:22:26.030 DEBG up_ds_listen process 1091
Sep 22 23:22:26.030 DEBG [A] ack job 1091:92, : downstairs
Sep 22 23:22:26.031 TRCE incoming request, uri: /newextent/16/db, method: GET, req_id: d9385f59-6676-4a50-8fad-30f09f1d4ccf, remote_addr: 127.0.0.1:50664, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.032 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/16/db, method: GET, req_id: d9385f59-6676-4a50-8fad-30f09f1d4ccf, remote_addr: 127.0.0.1:50664, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.033 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/010.copy" to "/tmp/downstairs-zrMnlo6G/00/000/010.replace"
Sep 22 23:22:26.033 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.034 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/010.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.034 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/010"
Sep 22 23:22:26.034 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/010.db"
Sep 22 23:22:26.034 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.034 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/010.replace" to "/tmp/downstairs-zrMnlo6G/00/000/010.completed"
Sep 22 23:22:26.034 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.034 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.035 DEBG [0] It's time to notify for 606
Sep 22 23:22:26.035 INFO Completion from [0] id:606 status:true
Sep 22 23:22:26.035 INFO [607/752] Repair commands completed
Sep 22 23:22:26.035 INFO Pop front: ReconcileIO { id: ReconciliationId(607), op: ExtentReopen { repair_id: ReconciliationId(607), extent_id: 16 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.035 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.035 INFO [0] received reconcile message
Sep 22 23:22:26.035 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(607), op: ExtentReopen { repair_id: ReconciliationId(607), extent_id: 16 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.035 INFO [0] client ExtentReopen { repair_id: ReconciliationId(607), extent_id: 16 }
Sep 22 23:22:26.035 INFO [1] received reconcile message
Sep 22 23:22:26.035 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(607), op: ExtentReopen { repair_id: ReconciliationId(607), extent_id: 16 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.035 INFO [1] client ExtentReopen { repair_id: ReconciliationId(607), extent_id: 16 }
Sep 22 23:22:26.035 INFO [2] received reconcile message
Sep 22 23:22:26.035 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(607), op: ExtentReopen { repair_id: ReconciliationId(607), extent_id: 16 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.035 INFO [2] client ExtentReopen { repair_id: ReconciliationId(607), extent_id: 16 }
Sep 22 23:22:26.035 DEBG 607 Reopen extent 16
Sep 22 23:22:26.036 DEBG 607 Reopen extent 16
Sep 22 23:22:26.036 DEBG 607 Reopen extent 16
Sep 22 23:22:26.037 DEBG [2] It's time to notify for 607
Sep 22 23:22:26.037 INFO Completion from [2] id:607 status:true
Sep 22 23:22:26.037 INFO [608/752] Repair commands completed
Sep 22 23:22:26.037 INFO Pop front: ReconcileIO { id: ReconciliationId(608), op: ExtentFlush { repair_id: ReconciliationId(608), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.037 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.037 INFO [0] received reconcile message
Sep 22 23:22:26.037 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(608), op: ExtentFlush { repair_id: ReconciliationId(608), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.037 INFO [0] client ExtentFlush { repair_id: ReconciliationId(608), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.037 INFO [1] received reconcile message
Sep 22 23:22:26.037 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(608), op: ExtentFlush { repair_id: ReconciliationId(608), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.037 INFO [1] client ExtentFlush { repair_id: ReconciliationId(608), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.037 INFO [2] received reconcile message
Sep 22 23:22:26.037 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(608), op: ExtentFlush { repair_id: ReconciliationId(608), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.037 INFO [2] client ExtentFlush { repair_id: ReconciliationId(608), extent_id: 4, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.037 DEBG 608 Flush extent 4 with f:2 g:2
Sep 22 23:22:26.037 DEBG Flush just extent 4 with f:2 and g:2
Sep 22 23:22:26.038 DEBG [1] It's time to notify for 608
Sep 22 23:22:26.038 INFO Completion from [1] id:608 status:true
Sep 22 23:22:26.038 INFO [609/752] Repair commands completed
Sep 22 23:22:26.038 INFO Pop front: ReconcileIO { id: ReconciliationId(609), op: ExtentClose { repair_id: ReconciliationId(609), extent_id: 4 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.038 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.038 INFO [0] received reconcile message
Sep 22 23:22:26.038 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(609), op: ExtentClose { repair_id: ReconciliationId(609), extent_id: 4 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.038 INFO [0] client ExtentClose { repair_id: ReconciliationId(609), extent_id: 4 }
Sep 22 23:22:26.038 INFO [1] received reconcile message
Sep 22 23:22:26.038 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(609), op: ExtentClose { repair_id: ReconciliationId(609), extent_id: 4 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.038 INFO [1] client ExtentClose { repair_id: ReconciliationId(609), extent_id: 4 }
Sep 22 23:22:26.038 INFO [2] received reconcile message
Sep 22 23:22:26.038 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(609), op: ExtentClose { repair_id: ReconciliationId(609), extent_id: 4 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.038 INFO [2] client ExtentClose { repair_id: ReconciliationId(609), extent_id: 4 }
Sep 22 23:22:26.038 DEBG 609 Close extent 4
Sep 22 23:22:26.038 DEBG 609 Close extent 4
Sep 22 23:22:26.039 DEBG 609 Close extent 4
Sep 22 23:22:26.039 DEBG [2] It's time to notify for 609
Sep 22 23:22:26.039 INFO Completion from [2] id:609 status:true
Sep 22 23:22:26.039 INFO [610/752] Repair commands completed
Sep 22 23:22:26.039 INFO Pop front: ReconcileIO { id: ReconciliationId(610), op: ExtentRepair { repair_id: ReconciliationId(610), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.039 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.039 INFO [0] received reconcile message
Sep 22 23:22:26.039 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(610), op: ExtentRepair { repair_id: ReconciliationId(610), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.039 INFO [0] client ExtentRepair { repair_id: ReconciliationId(610), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.039 INFO [0] Sending repair request ReconciliationId(610)
Sep 22 23:22:26.039 INFO [1] received reconcile message
Sep 22 23:22:26.039 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(610), op: ExtentRepair { repair_id: ReconciliationId(610), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.039 INFO [1] client ExtentRepair { repair_id: ReconciliationId(610), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.039 INFO [1] No action required ReconciliationId(610)
Sep 22 23:22:26.039 INFO [2] received reconcile message
Sep 22 23:22:26.039 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(610), op: ExtentRepair { repair_id: ReconciliationId(610), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.039 INFO [2] client ExtentRepair { repair_id: ReconciliationId(610), extent_id: 4, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.039 INFO [2] No action required ReconciliationId(610)
Sep 22 23:22:26.040 DEBG 610 Repair extent 4 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.040 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/004.copy"
Sep 22 23:22:26.083 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:26.086 DEBG Flush     :1088 extent_limit None deps:[JobId(1087), JobId(1086)] res:true f:33 g:1
Sep 22 23:22:26.086 WARN returning error on read!
Sep 22 23:22:26.086 DEBG Read      :1089 deps:[JobId(1088)] res:false
Sep 22 23:22:26.086 INFO [lossy] skipping 1091
Sep 22 23:22:26.092 DEBG Read      :1089 deps:[JobId(1088)] res:true
Sep 22 23:22:26.103 INFO accepted connection, remote_addr: 127.0.0.1:33331, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.104 TRCE incoming request, uri: /extent/4/files, method: GET, req_id: 83604101-67d1-4d48-a6d0-00ccd22ed211, remote_addr: 127.0.0.1:33331, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.104 INFO request completed, latency_us: 216, response_code: 200, uri: /extent/4/files, method: GET, req_id: 83604101-67d1-4d48-a6d0-00ccd22ed211, remote_addr: 127.0.0.1:33331, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.104 INFO eid:4 Found repair files: ["004", "004.db"]
Sep 22 23:22:26.104 TRCE incoming request, uri: /newextent/4/data, method: GET, req_id: 2f909e4b-ee48-4827-8390-535673e5c372, remote_addr: 127.0.0.1:33331, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.105 INFO request completed, latency_us: 335, response_code: 200, uri: /newextent/4/data, method: GET, req_id: 2f909e4b-ee48-4827-8390-535673e5c372, remote_addr: 127.0.0.1:33331, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.110 TRCE incoming request, uri: /newextent/4/db, method: GET, req_id: 9b352f7c-38d0-4f8a-9e58-959ad055ca33, remote_addr: 127.0.0.1:33331, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.110 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/4/db, method: GET, req_id: 9b352f7c-38d0-4f8a-9e58-959ad055ca33, remote_addr: 127.0.0.1:33331, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.111 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/004.copy" to "/tmp/downstairs-zrMnlo6G/00/000/004.replace"
Sep 22 23:22:26.111 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.112 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/004.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.112 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/004"
Sep 22 23:22:26.112 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/004.db"
Sep 22 23:22:26.112 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.112 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/004.replace" to "/tmp/downstairs-zrMnlo6G/00/000/004.completed"
Sep 22 23:22:26.112 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.112 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.113 DEBG [0] It's time to notify for 610
Sep 22 23:22:26.113 INFO Completion from [0] id:610 status:true
Sep 22 23:22:26.113 INFO [611/752] Repair commands completed
Sep 22 23:22:26.113 INFO Pop front: ReconcileIO { id: ReconciliationId(611), op: ExtentReopen { repair_id: ReconciliationId(611), extent_id: 4 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.113 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.113 INFO [0] received reconcile message
Sep 22 23:22:26.113 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(611), op: ExtentReopen { repair_id: ReconciliationId(611), extent_id: 4 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.113 INFO [0] client ExtentReopen { repair_id: ReconciliationId(611), extent_id: 4 }
Sep 22 23:22:26.113 INFO [1] received reconcile message
Sep 22 23:22:26.113 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(611), op: ExtentReopen { repair_id: ReconciliationId(611), extent_id: 4 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.113 INFO [1] client ExtentReopen { repair_id: ReconciliationId(611), extent_id: 4 }
Sep 22 23:22:26.113 INFO [2] received reconcile message
Sep 22 23:22:26.113 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(611), op: ExtentReopen { repair_id: ReconciliationId(611), extent_id: 4 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.113 INFO [2] client ExtentReopen { repair_id: ReconciliationId(611), extent_id: 4 }
Sep 22 23:22:26.113 DEBG 611 Reopen extent 4
Sep 22 23:22:26.114 DEBG IO Flush 1092 has deps [JobId(1091), JobId(1090)]
Sep 22 23:22:26.114 DEBG 611 Reopen extent 4
Sep 22 23:22:26.114 DEBG 611 Reopen extent 4
Sep 22 23:22:26.115 DEBG [2] It's time to notify for 611
Sep 22 23:22:26.115 INFO Completion from [2] id:611 status:true
Sep 22 23:22:26.115 INFO [612/752] Repair commands completed
Sep 22 23:22:26.115 INFO Pop front: ReconcileIO { id: ReconciliationId(612), op: ExtentFlush { repair_id: ReconciliationId(612), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.115 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.115 INFO [0] received reconcile message
Sep 22 23:22:26.115 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(612), op: ExtentFlush { repair_id: ReconciliationId(612), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.115 INFO [0] client ExtentFlush { repair_id: ReconciliationId(612), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.115 INFO [1] received reconcile message
Sep 22 23:22:26.115 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(612), op: ExtentFlush { repair_id: ReconciliationId(612), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.115 INFO [1] client ExtentFlush { repair_id: ReconciliationId(612), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.115 INFO [2] received reconcile message
Sep 22 23:22:26.115 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(612), op: ExtentFlush { repair_id: ReconciliationId(612), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.115 INFO [2] client ExtentFlush { repair_id: ReconciliationId(612), extent_id: 70, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.116 DEBG 612 Flush extent 70 with f:2 g:2
Sep 22 23:22:26.116 DEBG Flush just extent 70 with f:2 and g:2
Sep 22 23:22:26.116 DEBG [1] It's time to notify for 612
Sep 22 23:22:26.116 INFO Completion from [1] id:612 status:true
Sep 22 23:22:26.116 INFO [613/752] Repair commands completed
Sep 22 23:22:26.116 INFO Pop front: ReconcileIO { id: ReconciliationId(613), op: ExtentClose { repair_id: ReconciliationId(613), extent_id: 70 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.116 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.116 INFO [0] received reconcile message
Sep 22 23:22:26.116 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(613), op: ExtentClose { repair_id: ReconciliationId(613), extent_id: 70 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.116 INFO [0] client ExtentClose { repair_id: ReconciliationId(613), extent_id: 70 }
Sep 22 23:22:26.116 INFO [1] received reconcile message
Sep 22 23:22:26.116 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(613), op: ExtentClose { repair_id: ReconciliationId(613), extent_id: 70 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.116 INFO [1] client ExtentClose { repair_id: ReconciliationId(613), extent_id: 70 }
Sep 22 23:22:26.116 INFO [2] received reconcile message
Sep 22 23:22:26.116 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(613), op: ExtentClose { repair_id: ReconciliationId(613), extent_id: 70 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.116 INFO [2] client ExtentClose { repair_id: ReconciliationId(613), extent_id: 70 }
Sep 22 23:22:26.116 DEBG 613 Close extent 70
Sep 22 23:22:26.116 DEBG 613 Close extent 70
Sep 22 23:22:26.117 DEBG 613 Close extent 70
Sep 22 23:22:26.117 DEBG [2] It's time to notify for 613
Sep 22 23:22:26.117 INFO Completion from [2] id:613 status:true
Sep 22 23:22:26.117 INFO [614/752] Repair commands completed
Sep 22 23:22:26.117 INFO Pop front: ReconcileIO { id: ReconciliationId(614), op: ExtentRepair { repair_id: ReconciliationId(614), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.117 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.117 INFO [0] received reconcile message
Sep 22 23:22:26.117 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(614), op: ExtentRepair { repair_id: ReconciliationId(614), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.117 INFO [0] client ExtentRepair { repair_id: ReconciliationId(614), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.117 INFO [0] Sending repair request ReconciliationId(614)
Sep 22 23:22:26.117 INFO [1] received reconcile message
Sep 22 23:22:26.117 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(614), op: ExtentRepair { repair_id: ReconciliationId(614), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.117 INFO [1] client ExtentRepair { repair_id: ReconciliationId(614), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.117 INFO [1] No action required ReconciliationId(614)
Sep 22 23:22:26.117 INFO [2] received reconcile message
Sep 22 23:22:26.117 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(614), op: ExtentRepair { repair_id: ReconciliationId(614), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.118 INFO [2] client ExtentRepair { repair_id: ReconciliationId(614), extent_id: 70, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.118 INFO [2] No action required ReconciliationId(614)
Sep 22 23:22:26.118 DEBG 614 Repair extent 70 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.118 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/046.copy"
Sep 22 23:22:26.120 DEBG Read      :1091 deps:[JobId(1090)] res:true
Sep 22 23:22:26.182 INFO accepted connection, remote_addr: 127.0.0.1:36676, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.183 TRCE incoming request, uri: /extent/70/files, method: GET, req_id: 67fb5f22-d2c0-4ba6-855a-2158f293324d, remote_addr: 127.0.0.1:36676, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.183 INFO request completed, latency_us: 208, response_code: 200, uri: /extent/70/files, method: GET, req_id: 67fb5f22-d2c0-4ba6-855a-2158f293324d, remote_addr: 127.0.0.1:36676, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.183 INFO eid:70 Found repair files: ["046", "046.db"]
Sep 22 23:22:26.183 TRCE incoming request, uri: /newextent/70/data, method: GET, req_id: 6118aca9-01fe-47f6-8df9-e8e653b5a34c, remote_addr: 127.0.0.1:36676, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.184 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/70/data, method: GET, req_id: 6118aca9-01fe-47f6-8df9-e8e653b5a34c, remote_addr: 127.0.0.1:36676, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.188 TRCE incoming request, uri: /newextent/70/db, method: GET, req_id: 77a0f105-2491-4bc4-96ba-f33bb7212483, remote_addr: 127.0.0.1:36676, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.189 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/70/db, method: GET, req_id: 77a0f105-2491-4bc4-96ba-f33bb7212483, remote_addr: 127.0.0.1:36676, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.190 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/046.copy" to "/tmp/downstairs-zrMnlo6G/00/000/046.replace"
Sep 22 23:22:26.190 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.191 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/046.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.191 DEBG IO Read  1093 has deps [JobId(1092)]
Sep 22 23:22:26.191 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/046"
Sep 22 23:22:26.191 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/046.db"
Sep 22 23:22:26.191 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.191 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/046.replace" to "/tmp/downstairs-zrMnlo6G/00/000/046.completed"
Sep 22 23:22:26.191 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.191 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.191 DEBG [0] It's time to notify for 614
Sep 22 23:22:26.192 INFO Completion from [0] id:614 status:true
Sep 22 23:22:26.192 INFO [615/752] Repair commands completed
Sep 22 23:22:26.192 INFO Pop front: ReconcileIO { id: ReconciliationId(615), op: ExtentReopen { repair_id: ReconciliationId(615), extent_id: 70 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.192 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.192 INFO [0] received reconcile message
Sep 22 23:22:26.192 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(615), op: ExtentReopen { repair_id: ReconciliationId(615), extent_id: 70 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.192 INFO [0] client ExtentReopen { repair_id: ReconciliationId(615), extent_id: 70 }
Sep 22 23:22:26.192 INFO [1] received reconcile message
Sep 22 23:22:26.192 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(615), op: ExtentReopen { repair_id: ReconciliationId(615), extent_id: 70 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.192 INFO [1] client ExtentReopen { repair_id: ReconciliationId(615), extent_id: 70 }
Sep 22 23:22:26.192 INFO [2] received reconcile message
Sep 22 23:22:26.192 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(615), op: ExtentReopen { repair_id: ReconciliationId(615), extent_id: 70 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.192 INFO [2] client ExtentReopen { repair_id: ReconciliationId(615), extent_id: 70 }
Sep 22 23:22:26.192 DEBG 615 Reopen extent 70
Sep 22 23:22:26.193 DEBG 615 Reopen extent 70
Sep 22 23:22:26.193 DEBG 615 Reopen extent 70
Sep 22 23:22:26.194 DEBG [2] It's time to notify for 615
Sep 22 23:22:26.194 INFO Completion from [2] id:615 status:true
Sep 22 23:22:26.194 INFO [616/752] Repair commands completed
Sep 22 23:22:26.194 INFO Pop front: ReconcileIO { id: ReconciliationId(616), op: ExtentFlush { repair_id: ReconciliationId(616), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.194 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.194 INFO [0] received reconcile message
Sep 22 23:22:26.194 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(616), op: ExtentFlush { repair_id: ReconciliationId(616), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.194 INFO [0] client ExtentFlush { repair_id: ReconciliationId(616), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.194 INFO [1] received reconcile message
Sep 22 23:22:26.194 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(616), op: ExtentFlush { repair_id: ReconciliationId(616), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.194 INFO [1] client ExtentFlush { repair_id: ReconciliationId(616), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.194 INFO [2] received reconcile message
Sep 22 23:22:26.194 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(616), op: ExtentFlush { repair_id: ReconciliationId(616), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.194 INFO [2] client ExtentFlush { repair_id: ReconciliationId(616), extent_id: 76, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.194 DEBG 616 Flush extent 76 with f:2 g:2
Sep 22 23:22:26.194 DEBG Flush just extent 76 with f:2 and g:2
Sep 22 23:22:26.194 DEBG [1] It's time to notify for 616
Sep 22 23:22:26.194 INFO Completion from [1] id:616 status:true
Sep 22 23:22:26.194 INFO [617/752] Repair commands completed
Sep 22 23:22:26.194 INFO Pop front: ReconcileIO { id: ReconciliationId(617), op: ExtentClose { repair_id: ReconciliationId(617), extent_id: 76 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.194 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.194 INFO [0] received reconcile message
Sep 22 23:22:26.194 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(617), op: ExtentClose { repair_id: ReconciliationId(617), extent_id: 76 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.194 INFO [0] client ExtentClose { repair_id: ReconciliationId(617), extent_id: 76 }
Sep 22 23:22:26.194 INFO [1] received reconcile message
Sep 22 23:22:26.195 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(617), op: ExtentClose { repair_id: ReconciliationId(617), extent_id: 76 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.195 INFO [1] client ExtentClose { repair_id: ReconciliationId(617), extent_id: 76 }
Sep 22 23:22:26.195 INFO [2] received reconcile message
Sep 22 23:22:26.195 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(617), op: ExtentClose { repair_id: ReconciliationId(617), extent_id: 76 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.195 INFO [2] client ExtentClose { repair_id: ReconciliationId(617), extent_id: 76 }
Sep 22 23:22:26.195 DEBG 617 Close extent 76
Sep 22 23:22:26.195 DEBG 617 Close extent 76
Sep 22 23:22:26.195 DEBG 617 Close extent 76
Sep 22 23:22:26.196 DEBG [2] It's time to notify for 617
Sep 22 23:22:26.196 INFO Completion from [2] id:617 status:true
Sep 22 23:22:26.196 INFO [618/752] Repair commands completed
Sep 22 23:22:26.196 INFO Pop front: ReconcileIO { id: ReconciliationId(618), op: ExtentRepair { repair_id: ReconciliationId(618), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.196 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.196 INFO [0] received reconcile message
Sep 22 23:22:26.196 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(618), op: ExtentRepair { repair_id: ReconciliationId(618), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.196 INFO [0] client ExtentRepair { repair_id: ReconciliationId(618), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.196 INFO [0] Sending repair request ReconciliationId(618)
Sep 22 23:22:26.196 INFO [1] received reconcile message
Sep 22 23:22:26.196 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(618), op: ExtentRepair { repair_id: ReconciliationId(618), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.196 INFO [1] client ExtentRepair { repair_id: ReconciliationId(618), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.196 INFO [1] No action required ReconciliationId(618)
Sep 22 23:22:26.196 INFO [2] received reconcile message
Sep 22 23:22:26.196 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(618), op: ExtentRepair { repair_id: ReconciliationId(618), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.196 INFO [2] client ExtentRepair { repair_id: ReconciliationId(618), extent_id: 76, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.196 INFO [2] No action required ReconciliationId(618)
Sep 22 23:22:26.196 DEBG 618 Repair extent 76 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.196 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/04C.copy"
Sep 22 23:22:26.262 INFO accepted connection, remote_addr: 127.0.0.1:40842, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.262 TRCE incoming request, uri: /extent/76/files, method: GET, req_id: d18f3259-1576-4bf3-9c8e-af09bb3a911d, remote_addr: 127.0.0.1:40842, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.262 INFO request completed, latency_us: 200, response_code: 200, uri: /extent/76/files, method: GET, req_id: d18f3259-1576-4bf3-9c8e-af09bb3a911d, remote_addr: 127.0.0.1:40842, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.263 INFO eid:76 Found repair files: ["04C", "04C.db"]
Sep 22 23:22:26.263 TRCE incoming request, uri: /newextent/76/data, method: GET, req_id: f99fadc7-eff6-4028-983c-e2503292abb5, remote_addr: 127.0.0.1:40842, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.263 INFO request completed, latency_us: 321, response_code: 200, uri: /newextent/76/data, method: GET, req_id: f99fadc7-eff6-4028-983c-e2503292abb5, remote_addr: 127.0.0.1:40842, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.268 TRCE incoming request, uri: /newextent/76/db, method: GET, req_id: 6a537014-bbea-46e8-b972-43c3a9367c42, remote_addr: 127.0.0.1:40842, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.268 INFO request completed, latency_us: 298, response_code: 200, uri: /newextent/76/db, method: GET, req_id: 6a537014-bbea-46e8-b972-43c3a9367c42, remote_addr: 127.0.0.1:40842, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.269 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/04C.copy" to "/tmp/downstairs-zrMnlo6G/00/000/04C.replace"
Sep 22 23:22:26.269 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.270 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/04C.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.271 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04C"
Sep 22 23:22:26.271 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/04C.db"
Sep 22 23:22:26.271 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.271 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/04C.replace" to "/tmp/downstairs-zrMnlo6G/00/000/04C.completed"
Sep 22 23:22:26.271 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.271 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.271 DEBG [0] It's time to notify for 618
Sep 22 23:22:26.271 INFO Completion from [0] id:618 status:true
Sep 22 23:22:26.271 INFO [619/752] Repair commands completed
Sep 22 23:22:26.271 INFO Pop front: ReconcileIO { id: ReconciliationId(619), op: ExtentReopen { repair_id: ReconciliationId(619), extent_id: 76 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.271 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.271 INFO [0] received reconcile message
Sep 22 23:22:26.271 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(619), op: ExtentReopen { repair_id: ReconciliationId(619), extent_id: 76 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.271 INFO [0] client ExtentReopen { repair_id: ReconciliationId(619), extent_id: 76 }
Sep 22 23:22:26.271 INFO [1] received reconcile message
Sep 22 23:22:26.271 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(619), op: ExtentReopen { repair_id: ReconciliationId(619), extent_id: 76 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.271 INFO [1] client ExtentReopen { repair_id: ReconciliationId(619), extent_id: 76 }
Sep 22 23:22:26.271 INFO [2] received reconcile message
Sep 22 23:22:26.271 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(619), op: ExtentReopen { repair_id: ReconciliationId(619), extent_id: 76 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.271 INFO [2] client ExtentReopen { repair_id: ReconciliationId(619), extent_id: 76 }
Sep 22 23:22:26.272 DEBG 619 Reopen extent 76
Sep 22 23:22:26.272 DEBG 619 Reopen extent 76
Sep 22 23:22:26.273 DEBG 619 Reopen extent 76
Sep 22 23:22:26.273 DEBG [2] It's time to notify for 619
Sep 22 23:22:26.273 INFO Completion from [2] id:619 status:true
Sep 22 23:22:26.273 INFO [620/752] Repair commands completed
Sep 22 23:22:26.273 INFO Pop front: ReconcileIO { id: ReconciliationId(620), op: ExtentFlush { repair_id: ReconciliationId(620), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.273 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.273 INFO [0] received reconcile message
Sep 22 23:22:26.273 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(620), op: ExtentFlush { repair_id: ReconciliationId(620), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.273 INFO [0] client ExtentFlush { repair_id: ReconciliationId(620), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.273 INFO [1] received reconcile message
Sep 22 23:22:26.273 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(620), op: ExtentFlush { repair_id: ReconciliationId(620), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.273 INFO [1] client ExtentFlush { repair_id: ReconciliationId(620), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.273 INFO [2] received reconcile message
Sep 22 23:22:26.273 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(620), op: ExtentFlush { repair_id: ReconciliationId(620), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.274 INFO [2] client ExtentFlush { repair_id: ReconciliationId(620), extent_id: 113, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.274 DEBG 620 Flush extent 113 with f:2 g:2
Sep 22 23:22:26.274 DEBG Flush just extent 113 with f:2 and g:2
Sep 22 23:22:26.274 DEBG [1] It's time to notify for 620
Sep 22 23:22:26.274 INFO Completion from [1] id:620 status:true
Sep 22 23:22:26.274 INFO [621/752] Repair commands completed
Sep 22 23:22:26.274 INFO Pop front: ReconcileIO { id: ReconciliationId(621), op: ExtentClose { repair_id: ReconciliationId(621), extent_id: 113 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.274 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.274 INFO [0] received reconcile message
Sep 22 23:22:26.274 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(621), op: ExtentClose { repair_id: ReconciliationId(621), extent_id: 113 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.274 INFO [0] client ExtentClose { repair_id: ReconciliationId(621), extent_id: 113 }
Sep 22 23:22:26.274 INFO [1] received reconcile message
Sep 22 23:22:26.274 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(621), op: ExtentClose { repair_id: ReconciliationId(621), extent_id: 113 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.274 INFO [1] client ExtentClose { repair_id: ReconciliationId(621), extent_id: 113 }
Sep 22 23:22:26.274 INFO [2] received reconcile message
Sep 22 23:22:26.274 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(621), op: ExtentClose { repair_id: ReconciliationId(621), extent_id: 113 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.274 INFO [2] client ExtentClose { repair_id: ReconciliationId(621), extent_id: 113 }
Sep 22 23:22:26.274 DEBG 621 Close extent 113
Sep 22 23:22:26.274 DEBG 621 Close extent 113
Sep 22 23:22:26.275 DEBG 621 Close extent 113
Sep 22 23:22:26.275 DEBG [2] It's time to notify for 621
Sep 22 23:22:26.275 INFO Completion from [2] id:621 status:true
Sep 22 23:22:26.275 INFO [622/752] Repair commands completed
Sep 22 23:22:26.275 INFO Pop front: ReconcileIO { id: ReconciliationId(622), op: ExtentRepair { repair_id: ReconciliationId(622), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.275 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.275 INFO [0] received reconcile message
Sep 22 23:22:26.275 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(622), op: ExtentRepair { repair_id: ReconciliationId(622), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.275 INFO [0] client ExtentRepair { repair_id: ReconciliationId(622), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.275 INFO [0] Sending repair request ReconciliationId(622)
Sep 22 23:22:26.275 INFO [1] received reconcile message
Sep 22 23:22:26.275 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(622), op: ExtentRepair { repair_id: ReconciliationId(622), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.275 INFO [1] client ExtentRepair { repair_id: ReconciliationId(622), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.275 INFO [1] No action required ReconciliationId(622)
Sep 22 23:22:26.276 INFO [2] received reconcile message
Sep 22 23:22:26.276 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(622), op: ExtentRepair { repair_id: ReconciliationId(622), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.276 INFO [2] client ExtentRepair { repair_id: ReconciliationId(622), extent_id: 113, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.276 INFO [2] No action required ReconciliationId(622)
Sep 22 23:22:26.276 DEBG 622 Repair extent 113 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.276 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/071.copy"
Sep 22 23:22:26.340 INFO accepted connection, remote_addr: 127.0.0.1:49516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.341 TRCE incoming request, uri: /extent/113/files, method: GET, req_id: cdbef51d-a593-4262-8312-e01cab453433, remote_addr: 127.0.0.1:49516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.341 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/113/files, method: GET, req_id: cdbef51d-a593-4262-8312-e01cab453433, remote_addr: 127.0.0.1:49516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.341 INFO eid:113 Found repair files: ["071", "071.db"]
Sep 22 23:22:26.341 TRCE incoming request, uri: /newextent/113/data, method: GET, req_id: aeb9ca80-ef5f-45e8-baf6-77a32111f503, remote_addr: 127.0.0.1:49516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.342 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/113/data, method: GET, req_id: aeb9ca80-ef5f-45e8-baf6-77a32111f503, remote_addr: 127.0.0.1:49516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.346 TRCE incoming request, uri: /newextent/113/db, method: GET, req_id: fe517cf5-bd5c-4a70-ad5e-c7994609d22b, remote_addr: 127.0.0.1:49516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.347 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/113/db, method: GET, req_id: fe517cf5-bd5c-4a70-ad5e-c7994609d22b, remote_addr: 127.0.0.1:49516, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.348 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/071.copy" to "/tmp/downstairs-zrMnlo6G/00/000/071.replace"
Sep 22 23:22:26.348 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.349 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/071.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.349 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/071"
Sep 22 23:22:26.349 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/071.db"
Sep 22 23:22:26.349 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.349 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/071.replace" to "/tmp/downstairs-zrMnlo6G/00/000/071.completed"
Sep 22 23:22:26.349 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.349 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.349 DEBG [0] It's time to notify for 622
Sep 22 23:22:26.349 INFO Completion from [0] id:622 status:true
Sep 22 23:22:26.349 INFO [623/752] Repair commands completed
Sep 22 23:22:26.350 INFO Pop front: ReconcileIO { id: ReconciliationId(623), op: ExtentReopen { repair_id: ReconciliationId(623), extent_id: 113 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.350 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.350 INFO [0] received reconcile message
Sep 22 23:22:26.350 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(623), op: ExtentReopen { repair_id: ReconciliationId(623), extent_id: 113 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.350 INFO [0] client ExtentReopen { repair_id: ReconciliationId(623), extent_id: 113 }
Sep 22 23:22:26.350 INFO [1] received reconcile message
Sep 22 23:22:26.350 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(623), op: ExtentReopen { repair_id: ReconciliationId(623), extent_id: 113 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.350 INFO [1] client ExtentReopen { repair_id: ReconciliationId(623), extent_id: 113 }
Sep 22 23:22:26.350 INFO [2] received reconcile message
Sep 22 23:22:26.350 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(623), op: ExtentReopen { repair_id: ReconciliationId(623), extent_id: 113 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.350 INFO [2] client ExtentReopen { repair_id: ReconciliationId(623), extent_id: 113 }
Sep 22 23:22:26.350 DEBG 623 Reopen extent 113
Sep 22 23:22:26.350 DEBG 623 Reopen extent 113
Sep 22 23:22:26.351 DEBG 623 Reopen extent 113
Sep 22 23:22:26.352 DEBG [2] It's time to notify for 623
Sep 22 23:22:26.352 INFO Completion from [2] id:623 status:true
Sep 22 23:22:26.352 INFO [624/752] Repair commands completed
Sep 22 23:22:26.352 INFO Pop front: ReconcileIO { id: ReconciliationId(624), op: ExtentFlush { repair_id: ReconciliationId(624), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.352 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.352 INFO [0] received reconcile message
Sep 22 23:22:26.352 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(624), op: ExtentFlush { repair_id: ReconciliationId(624), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.352 INFO [0] client ExtentFlush { repair_id: ReconciliationId(624), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.352 INFO [1] received reconcile message
Sep 22 23:22:26.352 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(624), op: ExtentFlush { repair_id: ReconciliationId(624), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.352 INFO [1] client ExtentFlush { repair_id: ReconciliationId(624), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.352 INFO [2] received reconcile message
Sep 22 23:22:26.352 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(624), op: ExtentFlush { repair_id: ReconciliationId(624), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.352 INFO [2] client ExtentFlush { repair_id: ReconciliationId(624), extent_id: 108, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.352 DEBG 624 Flush extent 108 with f:2 g:2
Sep 22 23:22:26.352 DEBG Flush just extent 108 with f:2 and g:2
Sep 22 23:22:26.352 DEBG [1] It's time to notify for 624
Sep 22 23:22:26.352 INFO Completion from [1] id:624 status:true
Sep 22 23:22:26.352 INFO [625/752] Repair commands completed
Sep 22 23:22:26.352 INFO Pop front: ReconcileIO { id: ReconciliationId(625), op: ExtentClose { repair_id: ReconciliationId(625), extent_id: 108 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.352 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.352 INFO [0] received reconcile message
Sep 22 23:22:26.352 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(625), op: ExtentClose { repair_id: ReconciliationId(625), extent_id: 108 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.352 INFO [0] client ExtentClose { repair_id: ReconciliationId(625), extent_id: 108 }
Sep 22 23:22:26.352 INFO [1] received reconcile message
Sep 22 23:22:26.352 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(625), op: ExtentClose { repair_id: ReconciliationId(625), extent_id: 108 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.352 INFO [1] client ExtentClose { repair_id: ReconciliationId(625), extent_id: 108 }
Sep 22 23:22:26.352 INFO [2] received reconcile message
Sep 22 23:22:26.352 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(625), op: ExtentClose { repair_id: ReconciliationId(625), extent_id: 108 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.353 INFO [2] client ExtentClose { repair_id: ReconciliationId(625), extent_id: 108 }
Sep 22 23:22:26.353 DEBG 625 Close extent 108
Sep 22 23:22:26.353 DEBG 625 Close extent 108
Sep 22 23:22:26.353 DEBG 625 Close extent 108
Sep 22 23:22:26.354 DEBG [2] It's time to notify for 625
Sep 22 23:22:26.354 INFO Completion from [2] id:625 status:true
Sep 22 23:22:26.354 INFO [626/752] Repair commands completed
Sep 22 23:22:26.354 INFO Pop front: ReconcileIO { id: ReconciliationId(626), op: ExtentRepair { repair_id: ReconciliationId(626), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.354 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.354 INFO [0] received reconcile message
Sep 22 23:22:26.354 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(626), op: ExtentRepair { repair_id: ReconciliationId(626), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.354 INFO [0] client ExtentRepair { repair_id: ReconciliationId(626), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.354 INFO [0] Sending repair request ReconciliationId(626)
Sep 22 23:22:26.354 INFO [1] received reconcile message
Sep 22 23:22:26.354 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(626), op: ExtentRepair { repair_id: ReconciliationId(626), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.354 INFO [1] client ExtentRepair { repair_id: ReconciliationId(626), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.354 INFO [1] No action required ReconciliationId(626)
Sep 22 23:22:26.354 INFO [2] received reconcile message
Sep 22 23:22:26.354 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(626), op: ExtentRepair { repair_id: ReconciliationId(626), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.354 INFO [2] client ExtentRepair { repair_id: ReconciliationId(626), extent_id: 108, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.354 INFO [2] No action required ReconciliationId(626)
Sep 22 23:22:26.354 DEBG 626 Repair extent 108 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.354 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/06C.copy"
Sep 22 23:22:26.418 INFO accepted connection, remote_addr: 127.0.0.1:40951, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.418 TRCE incoming request, uri: /extent/108/files, method: GET, req_id: b48acfff-076f-44d5-af5c-31280e28074a, remote_addr: 127.0.0.1:40951, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.418 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/108/files, method: GET, req_id: b48acfff-076f-44d5-af5c-31280e28074a, remote_addr: 127.0.0.1:40951, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.419 INFO eid:108 Found repair files: ["06C", "06C.db"]
Sep 22 23:22:26.419 TRCE incoming request, uri: /newextent/108/data, method: GET, req_id: eea66984-3fd3-454c-acb9-aab21b779a4f, remote_addr: 127.0.0.1:40951, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.419 INFO request completed, latency_us: 320, response_code: 200, uri: /newextent/108/data, method: GET, req_id: eea66984-3fd3-454c-acb9-aab21b779a4f, remote_addr: 127.0.0.1:40951, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.424 TRCE incoming request, uri: /newextent/108/db, method: GET, req_id: f55f3b33-285f-4e16-bc8b-1ff8f2684ed4, remote_addr: 127.0.0.1:40951, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.424 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/108/db, method: GET, req_id: f55f3b33-285f-4e16-bc8b-1ff8f2684ed4, remote_addr: 127.0.0.1:40951, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.425 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/06C.copy" to "/tmp/downstairs-zrMnlo6G/00/000/06C.replace"
Sep 22 23:22:26.425 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.426 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/06C.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.426 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06C"
Sep 22 23:22:26.427 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/06C.db"
Sep 22 23:22:26.427 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.427 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/06C.replace" to "/tmp/downstairs-zrMnlo6G/00/000/06C.completed"
Sep 22 23:22:26.427 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.427 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.427 DEBG [0] It's time to notify for 626
Sep 22 23:22:26.427 INFO Completion from [0] id:626 status:true
Sep 22 23:22:26.427 INFO [627/752] Repair commands completed
Sep 22 23:22:26.427 INFO Pop front: ReconcileIO { id: ReconciliationId(627), op: ExtentReopen { repair_id: ReconciliationId(627), extent_id: 108 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.427 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.427 INFO [0] received reconcile message
Sep 22 23:22:26.427 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(627), op: ExtentReopen { repair_id: ReconciliationId(627), extent_id: 108 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.427 INFO [0] client ExtentReopen { repair_id: ReconciliationId(627), extent_id: 108 }
Sep 22 23:22:26.427 INFO [1] received reconcile message
Sep 22 23:22:26.427 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(627), op: ExtentReopen { repair_id: ReconciliationId(627), extent_id: 108 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.427 INFO [1] client ExtentReopen { repair_id: ReconciliationId(627), extent_id: 108 }
Sep 22 23:22:26.427 INFO [2] received reconcile message
Sep 22 23:22:26.427 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(627), op: ExtentReopen { repair_id: ReconciliationId(627), extent_id: 108 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.427 INFO [2] client ExtentReopen { repair_id: ReconciliationId(627), extent_id: 108 }
Sep 22 23:22:26.427 DEBG 627 Reopen extent 108
Sep 22 23:22:26.428 DEBG 627 Reopen extent 108
Sep 22 23:22:26.428 DEBG 627 Reopen extent 108
Sep 22 23:22:26.429 DEBG [2] It's time to notify for 627
Sep 22 23:22:26.429 INFO Completion from [2] id:627 status:true
Sep 22 23:22:26.429 INFO [628/752] Repair commands completed
Sep 22 23:22:26.429 INFO Pop front: ReconcileIO { id: ReconciliationId(628), op: ExtentFlush { repair_id: ReconciliationId(628), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.429 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.429 INFO [0] received reconcile message
Sep 22 23:22:26.429 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(628), op: ExtentFlush { repair_id: ReconciliationId(628), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.429 INFO [0] client ExtentFlush { repair_id: ReconciliationId(628), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.429 INFO [1] received reconcile message
Sep 22 23:22:26.429 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(628), op: ExtentFlush { repair_id: ReconciliationId(628), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.429 INFO [1] client ExtentFlush { repair_id: ReconciliationId(628), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.429 INFO [2] received reconcile message
Sep 22 23:22:26.429 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(628), op: ExtentFlush { repair_id: ReconciliationId(628), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.429 INFO [2] client ExtentFlush { repair_id: ReconciliationId(628), extent_id: 42, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.429 DEBG 628 Flush extent 42 with f:2 g:2
Sep 22 23:22:26.430 DEBG Flush just extent 42 with f:2 and g:2
Sep 22 23:22:26.430 DEBG [1] It's time to notify for 628
Sep 22 23:22:26.430 INFO Completion from [1] id:628 status:true
Sep 22 23:22:26.430 INFO [629/752] Repair commands completed
Sep 22 23:22:26.430 INFO Pop front: ReconcileIO { id: ReconciliationId(629), op: ExtentClose { repair_id: ReconciliationId(629), extent_id: 42 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.430 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.430 INFO [0] received reconcile message
Sep 22 23:22:26.430 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(629), op: ExtentClose { repair_id: ReconciliationId(629), extent_id: 42 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.430 INFO [0] client ExtentClose { repair_id: ReconciliationId(629), extent_id: 42 }
Sep 22 23:22:26.430 INFO [1] received reconcile message
Sep 22 23:22:26.430 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(629), op: ExtentClose { repair_id: ReconciliationId(629), extent_id: 42 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.430 INFO [1] client ExtentClose { repair_id: ReconciliationId(629), extent_id: 42 }
Sep 22 23:22:26.430 INFO [2] received reconcile message
Sep 22 23:22:26.430 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(629), op: ExtentClose { repair_id: ReconciliationId(629), extent_id: 42 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.430 INFO [2] client ExtentClose { repair_id: ReconciliationId(629), extent_id: 42 }
Sep 22 23:22:26.430 DEBG 629 Close extent 42
Sep 22 23:22:26.430 DEBG 629 Close extent 42
Sep 22 23:22:26.431 DEBG 629 Close extent 42
Sep 22 23:22:26.431 DEBG [2] It's time to notify for 629
Sep 22 23:22:26.431 INFO Completion from [2] id:629 status:true
Sep 22 23:22:26.431 INFO [630/752] Repair commands completed
Sep 22 23:22:26.431 INFO Pop front: ReconcileIO { id: ReconciliationId(630), op: ExtentRepair { repair_id: ReconciliationId(630), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.431 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.431 INFO [0] received reconcile message
Sep 22 23:22:26.431 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(630), op: ExtentRepair { repair_id: ReconciliationId(630), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.431 INFO [0] client ExtentRepair { repair_id: ReconciliationId(630), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.431 INFO [0] Sending repair request ReconciliationId(630)
Sep 22 23:22:26.431 INFO [1] received reconcile message
Sep 22 23:22:26.431 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(630), op: ExtentRepair { repair_id: ReconciliationId(630), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.431 INFO [1] client ExtentRepair { repair_id: ReconciliationId(630), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.431 INFO [1] No action required ReconciliationId(630)
Sep 22 23:22:26.431 INFO [2] received reconcile message
Sep 22 23:22:26.431 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(630), op: ExtentRepair { repair_id: ReconciliationId(630), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.431 INFO [2] client ExtentRepair { repair_id: ReconciliationId(630), extent_id: 42, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.431 INFO [2] No action required ReconciliationId(630)
Sep 22 23:22:26.432 DEBG 630 Repair extent 42 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.432 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/02A.copy"
Sep 22 23:22:26.495 INFO accepted connection, remote_addr: 127.0.0.1:33983, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.495 TRCE incoming request, uri: /extent/42/files, method: GET, req_id: 315a7e9e-66e2-4a6e-ba44-4d7132c03715, remote_addr: 127.0.0.1:33983, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.495 INFO request completed, latency_us: 198, response_code: 200, uri: /extent/42/files, method: GET, req_id: 315a7e9e-66e2-4a6e-ba44-4d7132c03715, remote_addr: 127.0.0.1:33983, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.496 INFO eid:42 Found repair files: ["02A", "02A.db"]
Sep 22 23:22:26.496 TRCE incoming request, uri: /newextent/42/data, method: GET, req_id: 7669d07d-9705-4179-8c8e-dd39ba4c1a2e, remote_addr: 127.0.0.1:33983, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.496 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/42/data, method: GET, req_id: 7669d07d-9705-4179-8c8e-dd39ba4c1a2e, remote_addr: 127.0.0.1:33983, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.501 TRCE incoming request, uri: /newextent/42/db, method: GET, req_id: db000148-a40b-45e2-bb74-e067b1224381, remote_addr: 127.0.0.1:33983, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.501 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/42/db, method: GET, req_id: db000148-a40b-45e2-bb74-e067b1224381, remote_addr: 127.0.0.1:33983, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.502 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/02A.copy" to "/tmp/downstairs-zrMnlo6G/00/000/02A.replace"
Sep 22 23:22:26.502 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.503 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/02A.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.503 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02A"
Sep 22 23:22:26.503 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/02A.db"
Sep 22 23:22:26.503 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.503 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/02A.replace" to "/tmp/downstairs-zrMnlo6G/00/000/02A.completed"
Sep 22 23:22:26.503 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.504 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.504 DEBG [0] It's time to notify for 630
Sep 22 23:22:26.504 INFO Completion from [0] id:630 status:true
Sep 22 23:22:26.504 INFO [631/752] Repair commands completed
Sep 22 23:22:26.504 INFO Pop front: ReconcileIO { id: ReconciliationId(631), op: ExtentReopen { repair_id: ReconciliationId(631), extent_id: 42 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.504 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.504 INFO [0] received reconcile message
Sep 22 23:22:26.504 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(631), op: ExtentReopen { repair_id: ReconciliationId(631), extent_id: 42 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.504 INFO [0] client ExtentReopen { repair_id: ReconciliationId(631), extent_id: 42 }
Sep 22 23:22:26.504 INFO [1] received reconcile message
Sep 22 23:22:26.504 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(631), op: ExtentReopen { repair_id: ReconciliationId(631), extent_id: 42 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.504 INFO [1] client ExtentReopen { repair_id: ReconciliationId(631), extent_id: 42 }
Sep 22 23:22:26.504 INFO [2] received reconcile message
Sep 22 23:22:26.504 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(631), op: ExtentReopen { repair_id: ReconciliationId(631), extent_id: 42 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.504 INFO [2] client ExtentReopen { repair_id: ReconciliationId(631), extent_id: 42 }
Sep 22 23:22:26.504 DEBG 631 Reopen extent 42
Sep 22 23:22:26.505 DEBG 631 Reopen extent 42
Sep 22 23:22:26.505 DEBG 631 Reopen extent 42
Sep 22 23:22:26.506 DEBG [2] It's time to notify for 631
Sep 22 23:22:26.506 INFO Completion from [2] id:631 status:true
Sep 22 23:22:26.506 INFO [632/752] Repair commands completed
Sep 22 23:22:26.506 INFO Pop front: ReconcileIO { id: ReconciliationId(632), op: ExtentFlush { repair_id: ReconciliationId(632), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.506 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.506 INFO [0] received reconcile message
Sep 22 23:22:26.506 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(632), op: ExtentFlush { repair_id: ReconciliationId(632), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.506 INFO [0] client ExtentFlush { repair_id: ReconciliationId(632), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.506 INFO [1] received reconcile message
Sep 22 23:22:26.506 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(632), op: ExtentFlush { repair_id: ReconciliationId(632), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.506 INFO [1] client ExtentFlush { repair_id: ReconciliationId(632), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.506 INFO [2] received reconcile message
Sep 22 23:22:26.506 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(632), op: ExtentFlush { repair_id: ReconciliationId(632), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.506 INFO [2] client ExtentFlush { repair_id: ReconciliationId(632), extent_id: 140, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.506 DEBG 632 Flush extent 140 with f:2 g:2
Sep 22 23:22:26.506 DEBG Flush just extent 140 with f:2 and g:2
Sep 22 23:22:26.507 DEBG [1] It's time to notify for 632
Sep 22 23:22:26.507 INFO Completion from [1] id:632 status:true
Sep 22 23:22:26.507 INFO [633/752] Repair commands completed
Sep 22 23:22:26.507 INFO Pop front: ReconcileIO { id: ReconciliationId(633), op: ExtentClose { repair_id: ReconciliationId(633), extent_id: 140 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.507 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.507 INFO [0] received reconcile message
Sep 22 23:22:26.507 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(633), op: ExtentClose { repair_id: ReconciliationId(633), extent_id: 140 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.507 INFO [0] client ExtentClose { repair_id: ReconciliationId(633), extent_id: 140 }
Sep 22 23:22:26.507 INFO [1] received reconcile message
Sep 22 23:22:26.507 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(633), op: ExtentClose { repair_id: ReconciliationId(633), extent_id: 140 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.507 INFO [1] client ExtentClose { repair_id: ReconciliationId(633), extent_id: 140 }
Sep 22 23:22:26.507 INFO [2] received reconcile message
Sep 22 23:22:26.507 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(633), op: ExtentClose { repair_id: ReconciliationId(633), extent_id: 140 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.507 INFO [2] client ExtentClose { repair_id: ReconciliationId(633), extent_id: 140 }
Sep 22 23:22:26.507 DEBG 633 Close extent 140
Sep 22 23:22:26.507 DEBG 633 Close extent 140
Sep 22 23:22:26.508 DEBG 633 Close extent 140
Sep 22 23:22:26.508 DEBG [2] It's time to notify for 633
Sep 22 23:22:26.508 INFO Completion from [2] id:633 status:true
Sep 22 23:22:26.508 INFO [634/752] Repair commands completed
Sep 22 23:22:26.508 INFO Pop front: ReconcileIO { id: ReconciliationId(634), op: ExtentRepair { repair_id: ReconciliationId(634), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.508 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.508 INFO [0] received reconcile message
Sep 22 23:22:26.508 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(634), op: ExtentRepair { repair_id: ReconciliationId(634), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.508 INFO [0] client ExtentRepair { repair_id: ReconciliationId(634), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.508 INFO [0] Sending repair request ReconciliationId(634)
Sep 22 23:22:26.508 INFO [1] received reconcile message
Sep 22 23:22:26.508 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(634), op: ExtentRepair { repair_id: ReconciliationId(634), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.508 INFO [1] client ExtentRepair { repair_id: ReconciliationId(634), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.508 INFO [1] No action required ReconciliationId(634)
Sep 22 23:22:26.508 INFO [2] received reconcile message
Sep 22 23:22:26.508 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(634), op: ExtentRepair { repair_id: ReconciliationId(634), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.508 INFO [2] client ExtentRepair { repair_id: ReconciliationId(634), extent_id: 140, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.508 INFO [2] No action required ReconciliationId(634)
Sep 22 23:22:26.508 DEBG 634 Repair extent 140 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.509 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/08C.copy"
Sep 22 23:22:26.524 DEBG [rc] retire 1088 clears [JobId(1087), JobId(1088)], : downstairs
Sep 22 23:22:26.524 ERRO [2] job id 1089 saw error GenericError("test error")
Sep 22 23:22:26.539 INFO [lossy] sleeping 1 second
Sep 22 23:22:26.540 INFO [lossy] skipping 1091
Sep 22 23:22:26.540 DEBG Flush     :1090 extent_limit None deps:[JobId(1089), JobId(1088)] res:true f:34 g:1
Sep 22 23:22:26.546 DEBG Read      :1091 deps:[JobId(1090)] res:true
Sep 22 23:22:26.574 INFO accepted connection, remote_addr: 127.0.0.1:44342, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.574 TRCE incoming request, uri: /extent/140/files, method: GET, req_id: dd918bdf-ab9d-47b1-a139-dcfeb07a9c06, remote_addr: 127.0.0.1:44342, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.574 INFO request completed, latency_us: 258, response_code: 200, uri: /extent/140/files, method: GET, req_id: dd918bdf-ab9d-47b1-a139-dcfeb07a9c06, remote_addr: 127.0.0.1:44342, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.575 INFO eid:140 Found repair files: ["08C", "08C.db"]
Sep 22 23:22:26.575 TRCE incoming request, uri: /newextent/140/data, method: GET, req_id: 5ce8f75a-1d0d-476f-8932-ddb5ec8d0025, remote_addr: 127.0.0.1:44342, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.575 INFO request completed, latency_us: 361, response_code: 200, uri: /newextent/140/data, method: GET, req_id: 5ce8f75a-1d0d-476f-8932-ddb5ec8d0025, remote_addr: 127.0.0.1:44342, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.580 TRCE incoming request, uri: /newextent/140/db, method: GET, req_id: bd1862d3-c3f5-47c9-97e5-d2ed282438aa, remote_addr: 127.0.0.1:44342, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.580 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/140/db, method: GET, req_id: bd1862d3-c3f5-47c9-97e5-d2ed282438aa, remote_addr: 127.0.0.1:44342, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.581 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/08C.copy" to "/tmp/downstairs-zrMnlo6G/00/000/08C.replace"
Sep 22 23:22:26.582 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.583 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/08C.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.583 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08C"
Sep 22 23:22:26.583 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08C.db"
Sep 22 23:22:26.583 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.583 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/08C.replace" to "/tmp/downstairs-zrMnlo6G/00/000/08C.completed"
Sep 22 23:22:26.583 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.583 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.583 DEBG [0] It's time to notify for 634
Sep 22 23:22:26.583 INFO Completion from [0] id:634 status:true
Sep 22 23:22:26.583 INFO [635/752] Repair commands completed
Sep 22 23:22:26.583 INFO Pop front: ReconcileIO { id: ReconciliationId(635), op: ExtentReopen { repair_id: ReconciliationId(635), extent_id: 140 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.584 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.584 INFO [0] received reconcile message
Sep 22 23:22:26.584 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(635), op: ExtentReopen { repair_id: ReconciliationId(635), extent_id: 140 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.584 INFO [0] client ExtentReopen { repair_id: ReconciliationId(635), extent_id: 140 }
Sep 22 23:22:26.584 INFO [1] received reconcile message
Sep 22 23:22:26.584 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(635), op: ExtentReopen { repair_id: ReconciliationId(635), extent_id: 140 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.584 INFO [1] client ExtentReopen { repair_id: ReconciliationId(635), extent_id: 140 }
Sep 22 23:22:26.584 INFO [2] received reconcile message
Sep 22 23:22:26.584 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(635), op: ExtentReopen { repair_id: ReconciliationId(635), extent_id: 140 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.584 INFO [2] client ExtentReopen { repair_id: ReconciliationId(635), extent_id: 140 }
Sep 22 23:22:26.584 DEBG 635 Reopen extent 140
Sep 22 23:22:26.585 DEBG 635 Reopen extent 140
Sep 22 23:22:26.585 DEBG 635 Reopen extent 140
Sep 22 23:22:26.586 DEBG [2] It's time to notify for 635
Sep 22 23:22:26.586 INFO Completion from [2] id:635 status:true
Sep 22 23:22:26.586 INFO [636/752] Repair commands completed
Sep 22 23:22:26.586 INFO Pop front: ReconcileIO { id: ReconciliationId(636), op: ExtentFlush { repair_id: ReconciliationId(636), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.586 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.586 INFO [0] received reconcile message
Sep 22 23:22:26.586 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(636), op: ExtentFlush { repair_id: ReconciliationId(636), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.586 INFO [0] client ExtentFlush { repair_id: ReconciliationId(636), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.586 INFO [1] received reconcile message
Sep 22 23:22:26.586 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(636), op: ExtentFlush { repair_id: ReconciliationId(636), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.586 INFO [1] client ExtentFlush { repair_id: ReconciliationId(636), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.586 INFO [2] received reconcile message
Sep 22 23:22:26.586 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(636), op: ExtentFlush { repair_id: ReconciliationId(636), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.586 INFO [2] client ExtentFlush { repair_id: ReconciliationId(636), extent_id: 178, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.586 DEBG 636 Flush extent 178 with f:2 g:2
Sep 22 23:22:26.586 DEBG Flush just extent 178 with f:2 and g:2
Sep 22 23:22:26.586 DEBG [1] It's time to notify for 636
Sep 22 23:22:26.586 INFO Completion from [1] id:636 status:true
Sep 22 23:22:26.586 INFO [637/752] Repair commands completed
Sep 22 23:22:26.586 INFO Pop front: ReconcileIO { id: ReconciliationId(637), op: ExtentClose { repair_id: ReconciliationId(637), extent_id: 178 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.587 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.587 INFO [0] received reconcile message
Sep 22 23:22:26.587 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(637), op: ExtentClose { repair_id: ReconciliationId(637), extent_id: 178 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.587 INFO [0] client ExtentClose { repair_id: ReconciliationId(637), extent_id: 178 }
Sep 22 23:22:26.587 INFO [1] received reconcile message
Sep 22 23:22:26.587 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(637), op: ExtentClose { repair_id: ReconciliationId(637), extent_id: 178 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.587 INFO [1] client ExtentClose { repair_id: ReconciliationId(637), extent_id: 178 }
Sep 22 23:22:26.587 INFO [2] received reconcile message
Sep 22 23:22:26.587 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(637), op: ExtentClose { repair_id: ReconciliationId(637), extent_id: 178 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.587 INFO [2] client ExtentClose { repair_id: ReconciliationId(637), extent_id: 178 }
Sep 22 23:22:26.587 DEBG 637 Close extent 178
Sep 22 23:22:26.587 DEBG 637 Close extent 178
Sep 22 23:22:26.587 DEBG 637 Close extent 178
Sep 22 23:22:26.588 DEBG [2] It's time to notify for 637
Sep 22 23:22:26.588 INFO Completion from [2] id:637 status:true
Sep 22 23:22:26.588 INFO [638/752] Repair commands completed
Sep 22 23:22:26.588 INFO Pop front: ReconcileIO { id: ReconciliationId(638), op: ExtentRepair { repair_id: ReconciliationId(638), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.588 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.588 INFO [0] received reconcile message
Sep 22 23:22:26.588 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(638), op: ExtentRepair { repair_id: ReconciliationId(638), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.588 INFO [0] client ExtentRepair { repair_id: ReconciliationId(638), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.588 INFO [0] Sending repair request ReconciliationId(638)
Sep 22 23:22:26.588 INFO [1] received reconcile message
Sep 22 23:22:26.588 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(638), op: ExtentRepair { repair_id: ReconciliationId(638), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.588 INFO [1] client ExtentRepair { repair_id: ReconciliationId(638), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.588 INFO [1] No action required ReconciliationId(638)
Sep 22 23:22:26.588 INFO [2] received reconcile message
Sep 22 23:22:26.588 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(638), op: ExtentRepair { repair_id: ReconciliationId(638), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.588 INFO [2] client ExtentRepair { repair_id: ReconciliationId(638), extent_id: 178, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.588 INFO [2] No action required ReconciliationId(638)
Sep 22 23:22:26.588 DEBG 638 Repair extent 178 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.588 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0B2.copy"
Sep 22 23:22:26.652 INFO accepted connection, remote_addr: 127.0.0.1:36395, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.653 TRCE incoming request, uri: /extent/178/files, method: GET, req_id: d15cee2d-0f33-4abd-b819-7e36806b3cbf, remote_addr: 127.0.0.1:36395, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.653 INFO request completed, latency_us: 265, response_code: 200, uri: /extent/178/files, method: GET, req_id: d15cee2d-0f33-4abd-b819-7e36806b3cbf, remote_addr: 127.0.0.1:36395, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.653 INFO eid:178 Found repair files: ["0B2", "0B2.db"]
Sep 22 23:22:26.654 TRCE incoming request, uri: /newextent/178/data, method: GET, req_id: fe8de25e-8017-44f2-97e8-4a1dc89e00fd, remote_addr: 127.0.0.1:36395, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.654 INFO request completed, latency_us: 324, response_code: 200, uri: /newextent/178/data, method: GET, req_id: fe8de25e-8017-44f2-97e8-4a1dc89e00fd, remote_addr: 127.0.0.1:36395, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.659 TRCE incoming request, uri: /newextent/178/db, method: GET, req_id: 754264d1-f973-40c0-a90b-ac4ba454aada, remote_addr: 127.0.0.1:36395, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.659 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/178/db, method: GET, req_id: 754264d1-f973-40c0-a90b-ac4ba454aada, remote_addr: 127.0.0.1:36395, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.660 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0B2.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0B2.replace"
Sep 22 23:22:26.660 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.662 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0B2.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.662 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B2"
Sep 22 23:22:26.662 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B2.db"
Sep 22 23:22:26.662 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.662 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0B2.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0B2.completed"
Sep 22 23:22:26.662 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.662 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.662 DEBG [0] It's time to notify for 638
Sep 22 23:22:26.662 INFO Completion from [0] id:638 status:true
Sep 22 23:22:26.663 INFO [639/752] Repair commands completed
Sep 22 23:22:26.663 INFO Pop front: ReconcileIO { id: ReconciliationId(639), op: ExtentReopen { repair_id: ReconciliationId(639), extent_id: 178 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.663 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.663 INFO [0] received reconcile message
Sep 22 23:22:26.663 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(639), op: ExtentReopen { repair_id: ReconciliationId(639), extent_id: 178 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.663 INFO [0] client ExtentReopen { repair_id: ReconciliationId(639), extent_id: 178 }
Sep 22 23:22:26.663 INFO [1] received reconcile message
Sep 22 23:22:26.663 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(639), op: ExtentReopen { repair_id: ReconciliationId(639), extent_id: 178 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.663 INFO [1] client ExtentReopen { repair_id: ReconciliationId(639), extent_id: 178 }
Sep 22 23:22:26.663 INFO [2] received reconcile message
Sep 22 23:22:26.663 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(639), op: ExtentReopen { repair_id: ReconciliationId(639), extent_id: 178 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.663 INFO [2] client ExtentReopen { repair_id: ReconciliationId(639), extent_id: 178 }
Sep 22 23:22:26.663 DEBG 639 Reopen extent 178
Sep 22 23:22:26.664 DEBG 639 Reopen extent 178
Sep 22 23:22:26.664 DEBG 639 Reopen extent 178
Sep 22 23:22:26.665 DEBG [2] It's time to notify for 639
Sep 22 23:22:26.665 INFO Completion from [2] id:639 status:true
Sep 22 23:22:26.665 INFO [640/752] Repair commands completed
Sep 22 23:22:26.665 INFO Pop front: ReconcileIO { id: ReconciliationId(640), op: ExtentFlush { repair_id: ReconciliationId(640), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.665 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.665 INFO [0] received reconcile message
Sep 22 23:22:26.665 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(640), op: ExtentFlush { repair_id: ReconciliationId(640), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.665 INFO [0] client ExtentFlush { repair_id: ReconciliationId(640), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.665 INFO [1] received reconcile message
Sep 22 23:22:26.665 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(640), op: ExtentFlush { repair_id: ReconciliationId(640), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.665 INFO [1] client ExtentFlush { repair_id: ReconciliationId(640), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.665 INFO [2] received reconcile message
Sep 22 23:22:26.665 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(640), op: ExtentFlush { repair_id: ReconciliationId(640), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.665 INFO [2] client ExtentFlush { repair_id: ReconciliationId(640), extent_id: 181, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.665 DEBG 640 Flush extent 181 with f:2 g:2
Sep 22 23:22:26.665 DEBG Flush just extent 181 with f:2 and g:2
Sep 22 23:22:26.665 DEBG [1] It's time to notify for 640
Sep 22 23:22:26.666 INFO Completion from [1] id:640 status:true
Sep 22 23:22:26.666 INFO [641/752] Repair commands completed
Sep 22 23:22:26.666 INFO Pop front: ReconcileIO { id: ReconciliationId(641), op: ExtentClose { repair_id: ReconciliationId(641), extent_id: 181 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.666 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.666 INFO [0] received reconcile message
Sep 22 23:22:26.666 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(641), op: ExtentClose { repair_id: ReconciliationId(641), extent_id: 181 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.666 INFO [0] client ExtentClose { repair_id: ReconciliationId(641), extent_id: 181 }
Sep 22 23:22:26.666 INFO [1] received reconcile message
Sep 22 23:22:26.666 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(641), op: ExtentClose { repair_id: ReconciliationId(641), extent_id: 181 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.666 INFO [1] client ExtentClose { repair_id: ReconciliationId(641), extent_id: 181 }
Sep 22 23:22:26.666 INFO [2] received reconcile message
Sep 22 23:22:26.666 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(641), op: ExtentClose { repair_id: ReconciliationId(641), extent_id: 181 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.666 INFO [2] client ExtentClose { repair_id: ReconciliationId(641), extent_id: 181 }
Sep 22 23:22:26.666 DEBG 641 Close extent 181
Sep 22 23:22:26.666 DEBG 641 Close extent 181
Sep 22 23:22:26.667 DEBG 641 Close extent 181
Sep 22 23:22:26.667 DEBG [2] It's time to notify for 641
Sep 22 23:22:26.667 INFO Completion from [2] id:641 status:true
Sep 22 23:22:26.667 INFO [642/752] Repair commands completed
Sep 22 23:22:26.667 INFO Pop front: ReconcileIO { id: ReconciliationId(642), op: ExtentRepair { repair_id: ReconciliationId(642), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.667 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.667 INFO [0] received reconcile message
Sep 22 23:22:26.667 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(642), op: ExtentRepair { repair_id: ReconciliationId(642), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.667 INFO [0] client ExtentRepair { repair_id: ReconciliationId(642), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.667 INFO [0] Sending repair request ReconciliationId(642)
Sep 22 23:22:26.667 INFO [1] received reconcile message
Sep 22 23:22:26.667 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(642), op: ExtentRepair { repair_id: ReconciliationId(642), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.667 INFO [1] client ExtentRepair { repair_id: ReconciliationId(642), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.667 INFO [1] No action required ReconciliationId(642)
Sep 22 23:22:26.667 INFO [2] received reconcile message
Sep 22 23:22:26.667 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(642), op: ExtentRepair { repair_id: ReconciliationId(642), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.667 INFO [2] client ExtentRepair { repair_id: ReconciliationId(642), extent_id: 181, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.667 INFO [2] No action required ReconciliationId(642)
Sep 22 23:22:26.667 DEBG 642 Repair extent 181 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.668 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0B5.copy"
Sep 22 23:22:26.731 INFO accepted connection, remote_addr: 127.0.0.1:38885, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.732 TRCE incoming request, uri: /extent/181/files, method: GET, req_id: 3c1c3b8a-a104-4c00-8a6d-077047e4496f, remote_addr: 127.0.0.1:38885, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.732 INFO request completed, latency_us: 283, response_code: 200, uri: /extent/181/files, method: GET, req_id: 3c1c3b8a-a104-4c00-8a6d-077047e4496f, remote_addr: 127.0.0.1:38885, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.732 INFO eid:181 Found repair files: ["0B5", "0B5.db"]
Sep 22 23:22:26.733 TRCE incoming request, uri: /newextent/181/data, method: GET, req_id: f0492bd6-6c63-4f33-b2cf-eddba7948333, remote_addr: 127.0.0.1:38885, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.733 INFO request completed, latency_us: 326, response_code: 200, uri: /newextent/181/data, method: GET, req_id: f0492bd6-6c63-4f33-b2cf-eddba7948333, remote_addr: 127.0.0.1:38885, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.738 TRCE incoming request, uri: /newextent/181/db, method: GET, req_id: 29afa013-65a5-4a51-a49a-cf67d1d6e345, remote_addr: 127.0.0.1:38885, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.738 INFO request completed, latency_us: 306, response_code: 200, uri: /newextent/181/db, method: GET, req_id: 29afa013-65a5-4a51-a49a-cf67d1d6e345, remote_addr: 127.0.0.1:38885, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.739 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0B5.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0B5.replace"
Sep 22 23:22:26.739 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.741 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0B5.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.741 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B5"
Sep 22 23:22:26.741 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B5.db"
Sep 22 23:22:26.741 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.741 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0B5.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0B5.completed"
Sep 22 23:22:26.741 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.741 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.741 DEBG [0] It's time to notify for 642
Sep 22 23:22:26.741 INFO Completion from [0] id:642 status:true
Sep 22 23:22:26.741 INFO [643/752] Repair commands completed
Sep 22 23:22:26.742 INFO Pop front: ReconcileIO { id: ReconciliationId(643), op: ExtentReopen { repair_id: ReconciliationId(643), extent_id: 181 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.742 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.742 INFO [0] received reconcile message
Sep 22 23:22:26.742 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(643), op: ExtentReopen { repair_id: ReconciliationId(643), extent_id: 181 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.742 INFO [0] client ExtentReopen { repair_id: ReconciliationId(643), extent_id: 181 }
Sep 22 23:22:26.742 INFO [1] received reconcile message
Sep 22 23:22:26.742 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(643), op: ExtentReopen { repair_id: ReconciliationId(643), extent_id: 181 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.742 INFO [1] client ExtentReopen { repair_id: ReconciliationId(643), extent_id: 181 }
Sep 22 23:22:26.742 INFO [2] received reconcile message
Sep 22 23:22:26.742 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(643), op: ExtentReopen { repair_id: ReconciliationId(643), extent_id: 181 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.742 INFO [2] client ExtentReopen { repair_id: ReconciliationId(643), extent_id: 181 }
Sep 22 23:22:26.742 DEBG 643 Reopen extent 181
Sep 22 23:22:26.743 DEBG 643 Reopen extent 181
Sep 22 23:22:26.743 DEBG 643 Reopen extent 181
Sep 22 23:22:26.744 DEBG [2] It's time to notify for 643
Sep 22 23:22:26.744 INFO Completion from [2] id:643 status:true
Sep 22 23:22:26.744 INFO [644/752] Repair commands completed
Sep 22 23:22:26.744 INFO Pop front: ReconcileIO { id: ReconciliationId(644), op: ExtentFlush { repair_id: ReconciliationId(644), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.744 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.744 INFO [0] received reconcile message
Sep 22 23:22:26.744 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(644), op: ExtentFlush { repair_id: ReconciliationId(644), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.744 INFO [0] client ExtentFlush { repair_id: ReconciliationId(644), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.744 INFO [1] received reconcile message
Sep 22 23:22:26.744 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(644), op: ExtentFlush { repair_id: ReconciliationId(644), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.744 INFO [1] client ExtentFlush { repair_id: ReconciliationId(644), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.744 INFO [2] received reconcile message
Sep 22 23:22:26.744 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(644), op: ExtentFlush { repair_id: ReconciliationId(644), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.744 INFO [2] client ExtentFlush { repair_id: ReconciliationId(644), extent_id: 82, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.744 DEBG 644 Flush extent 82 with f:2 g:2
Sep 22 23:22:26.744 DEBG Flush just extent 82 with f:2 and g:2
Sep 22 23:22:26.744 DEBG [1] It's time to notify for 644
Sep 22 23:22:26.745 INFO Completion from [1] id:644 status:true
Sep 22 23:22:26.745 INFO [645/752] Repair commands completed
Sep 22 23:22:26.745 INFO Pop front: ReconcileIO { id: ReconciliationId(645), op: ExtentClose { repair_id: ReconciliationId(645), extent_id: 82 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.745 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.745 INFO [0] received reconcile message
Sep 22 23:22:26.745 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(645), op: ExtentClose { repair_id: ReconciliationId(645), extent_id: 82 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.745 INFO [0] client ExtentClose { repair_id: ReconciliationId(645), extent_id: 82 }
Sep 22 23:22:26.745 INFO [1] received reconcile message
Sep 22 23:22:26.745 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(645), op: ExtentClose { repair_id: ReconciliationId(645), extent_id: 82 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.745 INFO [1] client ExtentClose { repair_id: ReconciliationId(645), extent_id: 82 }
Sep 22 23:22:26.745 INFO [2] received reconcile message
Sep 22 23:22:26.745 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(645), op: ExtentClose { repair_id: ReconciliationId(645), extent_id: 82 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.745 INFO [2] client ExtentClose { repair_id: ReconciliationId(645), extent_id: 82 }
Sep 22 23:22:26.745 DEBG 645 Close extent 82
Sep 22 23:22:26.745 DEBG 645 Close extent 82
Sep 22 23:22:26.746 DEBG 645 Close extent 82
Sep 22 23:22:26.746 DEBG [2] It's time to notify for 645
Sep 22 23:22:26.746 INFO Completion from [2] id:645 status:true
Sep 22 23:22:26.746 INFO [646/752] Repair commands completed
Sep 22 23:22:26.746 INFO Pop front: ReconcileIO { id: ReconciliationId(646), op: ExtentRepair { repair_id: ReconciliationId(646), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.746 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.746 INFO [0] received reconcile message
Sep 22 23:22:26.746 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(646), op: ExtentRepair { repair_id: ReconciliationId(646), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.746 INFO [0] client ExtentRepair { repair_id: ReconciliationId(646), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.746 INFO [0] Sending repair request ReconciliationId(646)
Sep 22 23:22:26.746 INFO [1] received reconcile message
Sep 22 23:22:26.746 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(646), op: ExtentRepair { repair_id: ReconciliationId(646), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.746 INFO [1] client ExtentRepair { repair_id: ReconciliationId(646), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.746 INFO [1] No action required ReconciliationId(646)
Sep 22 23:22:26.746 INFO [2] received reconcile message
Sep 22 23:22:26.746 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(646), op: ExtentRepair { repair_id: ReconciliationId(646), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.746 INFO [2] client ExtentRepair { repair_id: ReconciliationId(646), extent_id: 82, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.746 INFO [2] No action required ReconciliationId(646)
Sep 22 23:22:26.746 DEBG 646 Repair extent 82 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.747 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/052.copy"
Sep 22 23:22:26.812 INFO accepted connection, remote_addr: 127.0.0.1:38493, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.812 TRCE incoming request, uri: /extent/82/files, method: GET, req_id: 803b9bb7-8110-4e65-a5b7-73d5d0c75513, remote_addr: 127.0.0.1:38493, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.812 INFO request completed, latency_us: 206, response_code: 200, uri: /extent/82/files, method: GET, req_id: 803b9bb7-8110-4e65-a5b7-73d5d0c75513, remote_addr: 127.0.0.1:38493, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.813 INFO eid:82 Found repair files: ["052", "052.db"]
Sep 22 23:22:26.813 TRCE incoming request, uri: /newextent/82/data, method: GET, req_id: e9b4c0c9-6d3e-4465-a753-616b28d05957, remote_addr: 127.0.0.1:38493, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.813 INFO request completed, latency_us: 324, response_code: 200, uri: /newextent/82/data, method: GET, req_id: e9b4c0c9-6d3e-4465-a753-616b28d05957, remote_addr: 127.0.0.1:38493, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.818 TRCE incoming request, uri: /newextent/82/db, method: GET, req_id: aecea6b8-7f47-4609-8493-e862a8d94cf5, remote_addr: 127.0.0.1:38493, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.818 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/82/db, method: GET, req_id: aecea6b8-7f47-4609-8493-e862a8d94cf5, remote_addr: 127.0.0.1:38493, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.819 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/052.copy" to "/tmp/downstairs-zrMnlo6G/00/000/052.replace"
Sep 22 23:22:26.819 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.820 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/052.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.820 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/052"
Sep 22 23:22:26.820 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/052.db"
Sep 22 23:22:26.820 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.820 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/052.replace" to "/tmp/downstairs-zrMnlo6G/00/000/052.completed"
Sep 22 23:22:26.820 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.821 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.821 DEBG [0] It's time to notify for 646
Sep 22 23:22:26.821 INFO Completion from [0] id:646 status:true
Sep 22 23:22:26.821 INFO [647/752] Repair commands completed
Sep 22 23:22:26.821 INFO Pop front: ReconcileIO { id: ReconciliationId(647), op: ExtentReopen { repair_id: ReconciliationId(647), extent_id: 82 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.821 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.821 INFO [0] received reconcile message
Sep 22 23:22:26.821 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(647), op: ExtentReopen { repair_id: ReconciliationId(647), extent_id: 82 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.821 INFO [0] client ExtentReopen { repair_id: ReconciliationId(647), extent_id: 82 }
Sep 22 23:22:26.821 INFO [1] received reconcile message
Sep 22 23:22:26.821 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(647), op: ExtentReopen { repair_id: ReconciliationId(647), extent_id: 82 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.821 INFO [1] client ExtentReopen { repair_id: ReconciliationId(647), extent_id: 82 }
Sep 22 23:22:26.821 INFO [2] received reconcile message
Sep 22 23:22:26.821 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(647), op: ExtentReopen { repair_id: ReconciliationId(647), extent_id: 82 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.821 INFO [2] client ExtentReopen { repair_id: ReconciliationId(647), extent_id: 82 }
Sep 22 23:22:26.821 DEBG 647 Reopen extent 82
Sep 22 23:22:26.822 DEBG 647 Reopen extent 82
Sep 22 23:22:26.822 DEBG 647 Reopen extent 82
Sep 22 23:22:26.823 DEBG [2] It's time to notify for 647
Sep 22 23:22:26.823 INFO Completion from [2] id:647 status:true
Sep 22 23:22:26.823 INFO [648/752] Repair commands completed
Sep 22 23:22:26.823 INFO Pop front: ReconcileIO { id: ReconciliationId(648), op: ExtentFlush { repair_id: ReconciliationId(648), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.823 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.823 INFO [0] received reconcile message
Sep 22 23:22:26.823 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(648), op: ExtentFlush { repair_id: ReconciliationId(648), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.823 INFO [0] client ExtentFlush { repair_id: ReconciliationId(648), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.823 INFO [1] received reconcile message
Sep 22 23:22:26.823 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(648), op: ExtentFlush { repair_id: ReconciliationId(648), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.823 INFO [1] client ExtentFlush { repair_id: ReconciliationId(648), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.823 INFO [2] received reconcile message
Sep 22 23:22:26.823 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(648), op: ExtentFlush { repair_id: ReconciliationId(648), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.823 INFO [2] client ExtentFlush { repair_id: ReconciliationId(648), extent_id: 174, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.823 DEBG 648 Flush extent 174 with f:2 g:2
Sep 22 23:22:26.823 DEBG Flush just extent 174 with f:2 and g:2
Sep 22 23:22:26.824 DEBG [1] It's time to notify for 648
Sep 22 23:22:26.824 INFO Completion from [1] id:648 status:true
Sep 22 23:22:26.824 INFO [649/752] Repair commands completed
Sep 22 23:22:26.824 INFO Pop front: ReconcileIO { id: ReconciliationId(649), op: ExtentClose { repair_id: ReconciliationId(649), extent_id: 174 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.824 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.824 INFO [0] received reconcile message
Sep 22 23:22:26.824 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(649), op: ExtentClose { repair_id: ReconciliationId(649), extent_id: 174 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.824 INFO [0] client ExtentClose { repair_id: ReconciliationId(649), extent_id: 174 }
Sep 22 23:22:26.824 INFO [1] received reconcile message
Sep 22 23:22:26.824 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(649), op: ExtentClose { repair_id: ReconciliationId(649), extent_id: 174 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.824 INFO [1] client ExtentClose { repair_id: ReconciliationId(649), extent_id: 174 }
Sep 22 23:22:26.824 INFO [2] received reconcile message
Sep 22 23:22:26.824 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(649), op: ExtentClose { repair_id: ReconciliationId(649), extent_id: 174 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.824 INFO [2] client ExtentClose { repair_id: ReconciliationId(649), extent_id: 174 }
Sep 22 23:22:26.824 DEBG 649 Close extent 174
Sep 22 23:22:26.824 DEBG 649 Close extent 174
Sep 22 23:22:26.825 DEBG 649 Close extent 174
Sep 22 23:22:26.825 DEBG [2] It's time to notify for 649
Sep 22 23:22:26.825 INFO Completion from [2] id:649 status:true
Sep 22 23:22:26.825 INFO [650/752] Repair commands completed
Sep 22 23:22:26.825 INFO Pop front: ReconcileIO { id: ReconciliationId(650), op: ExtentRepair { repair_id: ReconciliationId(650), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.825 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.825 INFO [0] received reconcile message
Sep 22 23:22:26.825 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(650), op: ExtentRepair { repair_id: ReconciliationId(650), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.825 INFO [0] client ExtentRepair { repair_id: ReconciliationId(650), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.825 INFO [0] Sending repair request ReconciliationId(650)
Sep 22 23:22:26.825 INFO [1] received reconcile message
Sep 22 23:22:26.825 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(650), op: ExtentRepair { repair_id: ReconciliationId(650), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.825 INFO [1] client ExtentRepair { repair_id: ReconciliationId(650), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.825 INFO [1] No action required ReconciliationId(650)
Sep 22 23:22:26.825 INFO [2] received reconcile message
Sep 22 23:22:26.825 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(650), op: ExtentRepair { repair_id: ReconciliationId(650), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.825 INFO [2] client ExtentRepair { repair_id: ReconciliationId(650), extent_id: 174, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.825 INFO [2] No action required ReconciliationId(650)
Sep 22 23:22:26.826 DEBG 650 Repair extent 174 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.826 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0AE.copy"
Sep 22 23:22:26.891 INFO accepted connection, remote_addr: 127.0.0.1:32878, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.892 TRCE incoming request, uri: /extent/174/files, method: GET, req_id: 2e976354-b1d1-49e5-bba6-671cd2216493, remote_addr: 127.0.0.1:32878, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.892 INFO request completed, latency_us: 208, response_code: 200, uri: /extent/174/files, method: GET, req_id: 2e976354-b1d1-49e5-bba6-671cd2216493, remote_addr: 127.0.0.1:32878, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.892 INFO eid:174 Found repair files: ["0AE", "0AE.db"]
Sep 22 23:22:26.892 TRCE incoming request, uri: /newextent/174/data, method: GET, req_id: 5047da10-44ce-4508-944c-26526de03bd1, remote_addr: 127.0.0.1:32878, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.893 INFO request completed, latency_us: 307, response_code: 200, uri: /newextent/174/data, method: GET, req_id: 5047da10-44ce-4508-944c-26526de03bd1, remote_addr: 127.0.0.1:32878, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.897 TRCE incoming request, uri: /newextent/174/db, method: GET, req_id: 4e5a52d2-5002-4df2-a936-4a2ae13a46cd, remote_addr: 127.0.0.1:32878, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.898 INFO request completed, latency_us: 318, response_code: 200, uri: /newextent/174/db, method: GET, req_id: 4e5a52d2-5002-4df2-a936-4a2ae13a46cd, remote_addr: 127.0.0.1:32878, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.899 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0AE.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0AE.replace"
Sep 22 23:22:26.899 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.900 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0AE.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.900 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AE"
Sep 22 23:22:26.900 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AE.db"
Sep 22 23:22:26.900 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.900 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0AE.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0AE.completed"
Sep 22 23:22:26.900 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.900 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.900 DEBG [0] It's time to notify for 650
Sep 22 23:22:26.900 INFO Completion from [0] id:650 status:true
Sep 22 23:22:26.900 INFO [651/752] Repair commands completed
Sep 22 23:22:26.901 INFO Pop front: ReconcileIO { id: ReconciliationId(651), op: ExtentReopen { repair_id: ReconciliationId(651), extent_id: 174 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.901 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.901 INFO [0] received reconcile message
Sep 22 23:22:26.901 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(651), op: ExtentReopen { repair_id: ReconciliationId(651), extent_id: 174 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.901 INFO [0] client ExtentReopen { repair_id: ReconciliationId(651), extent_id: 174 }
Sep 22 23:22:26.901 INFO [1] received reconcile message
Sep 22 23:22:26.901 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(651), op: ExtentReopen { repair_id: ReconciliationId(651), extent_id: 174 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.901 INFO [1] client ExtentReopen { repair_id: ReconciliationId(651), extent_id: 174 }
Sep 22 23:22:26.901 INFO [2] received reconcile message
Sep 22 23:22:26.901 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(651), op: ExtentReopen { repair_id: ReconciliationId(651), extent_id: 174 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.901 INFO [2] client ExtentReopen { repair_id: ReconciliationId(651), extent_id: 174 }
Sep 22 23:22:26.901 DEBG 651 Reopen extent 174
Sep 22 23:22:26.902 DEBG 651 Reopen extent 174
Sep 22 23:22:26.902 DEBG 651 Reopen extent 174
Sep 22 23:22:26.903 DEBG [2] It's time to notify for 651
Sep 22 23:22:26.903 INFO Completion from [2] id:651 status:true
Sep 22 23:22:26.903 INFO [652/752] Repair commands completed
Sep 22 23:22:26.903 INFO Pop front: ReconcileIO { id: ReconciliationId(652), op: ExtentFlush { repair_id: ReconciliationId(652), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.903 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.903 INFO [0] received reconcile message
Sep 22 23:22:26.903 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(652), op: ExtentFlush { repair_id: ReconciliationId(652), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.903 INFO [0] client ExtentFlush { repair_id: ReconciliationId(652), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.903 INFO [1] received reconcile message
Sep 22 23:22:26.903 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(652), op: ExtentFlush { repair_id: ReconciliationId(652), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.903 INFO [1] client ExtentFlush { repair_id: ReconciliationId(652), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.903 INFO [2] received reconcile message
Sep 22 23:22:26.903 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(652), op: ExtentFlush { repair_id: ReconciliationId(652), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.903 INFO [2] client ExtentFlush { repair_id: ReconciliationId(652), extent_id: 17, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.903 DEBG 652 Flush extent 17 with f:2 g:2
Sep 22 23:22:26.903 DEBG Flush just extent 17 with f:2 and g:2
Sep 22 23:22:26.903 DEBG [1] It's time to notify for 652
Sep 22 23:22:26.903 INFO Completion from [1] id:652 status:true
Sep 22 23:22:26.903 INFO [653/752] Repair commands completed
Sep 22 23:22:26.903 INFO Pop front: ReconcileIO { id: ReconciliationId(653), op: ExtentClose { repair_id: ReconciliationId(653), extent_id: 17 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.903 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.903 INFO [0] received reconcile message
Sep 22 23:22:26.903 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(653), op: ExtentClose { repair_id: ReconciliationId(653), extent_id: 17 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.903 INFO [0] client ExtentClose { repair_id: ReconciliationId(653), extent_id: 17 }
Sep 22 23:22:26.903 INFO [1] received reconcile message
Sep 22 23:22:26.903 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(653), op: ExtentClose { repair_id: ReconciliationId(653), extent_id: 17 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.903 INFO [1] client ExtentClose { repair_id: ReconciliationId(653), extent_id: 17 }
Sep 22 23:22:26.903 INFO [2] received reconcile message
Sep 22 23:22:26.903 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(653), op: ExtentClose { repair_id: ReconciliationId(653), extent_id: 17 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.904 INFO [2] client ExtentClose { repair_id: ReconciliationId(653), extent_id: 17 }
Sep 22 23:22:26.904 DEBG 653 Close extent 17
Sep 22 23:22:26.904 DEBG 653 Close extent 17
Sep 22 23:22:26.904 DEBG 653 Close extent 17
Sep 22 23:22:26.905 DEBG [2] It's time to notify for 653
Sep 22 23:22:26.905 INFO Completion from [2] id:653 status:true
Sep 22 23:22:26.905 INFO [654/752] Repair commands completed
Sep 22 23:22:26.905 INFO Pop front: ReconcileIO { id: ReconciliationId(654), op: ExtentRepair { repair_id: ReconciliationId(654), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.905 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.905 INFO [0] received reconcile message
Sep 22 23:22:26.905 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(654), op: ExtentRepair { repair_id: ReconciliationId(654), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.905 INFO [0] client ExtentRepair { repair_id: ReconciliationId(654), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.905 INFO [0] Sending repair request ReconciliationId(654)
Sep 22 23:22:26.905 INFO [1] received reconcile message
Sep 22 23:22:26.905 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(654), op: ExtentRepair { repair_id: ReconciliationId(654), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.905 INFO [1] client ExtentRepair { repair_id: ReconciliationId(654), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.905 INFO [1] No action required ReconciliationId(654)
Sep 22 23:22:26.905 INFO [2] received reconcile message
Sep 22 23:22:26.905 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(654), op: ExtentRepair { repair_id: ReconciliationId(654), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.905 INFO [2] client ExtentRepair { repair_id: ReconciliationId(654), extent_id: 17, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.905 INFO [2] No action required ReconciliationId(654)
Sep 22 23:22:26.905 DEBG 654 Repair extent 17 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.905 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/011.copy"
Sep 22 23:22:26.968 INFO accepted connection, remote_addr: 127.0.0.1:55601, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.969 TRCE incoming request, uri: /extent/17/files, method: GET, req_id: 3dd08200-033d-4d1a-8a33-07b962a4d5f7, remote_addr: 127.0.0.1:55601, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.969 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/17/files, method: GET, req_id: 3dd08200-033d-4d1a-8a33-07b962a4d5f7, remote_addr: 127.0.0.1:55601, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.969 INFO eid:17 Found repair files: ["011", "011.db"]
Sep 22 23:22:26.969 TRCE incoming request, uri: /newextent/17/data, method: GET, req_id: ef664cc4-a80f-44e8-bd62-25d0276be673, remote_addr: 127.0.0.1:55601, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.970 INFO request completed, latency_us: 311, response_code: 200, uri: /newextent/17/data, method: GET, req_id: ef664cc4-a80f-44e8-bd62-25d0276be673, remote_addr: 127.0.0.1:55601, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.975 TRCE incoming request, uri: /newextent/17/db, method: GET, req_id: b3dd0017-0792-4a45-8259-0fb3a6bac94a, remote_addr: 127.0.0.1:55601, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.975 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/17/db, method: GET, req_id: b3dd0017-0792-4a45-8259-0fb3a6bac94a, remote_addr: 127.0.0.1:55601, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:26.976 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/011.copy" to "/tmp/downstairs-zrMnlo6G/00/000/011.replace"
Sep 22 23:22:26.976 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.977 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/011.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.977 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/011"
Sep 22 23:22:26.977 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/011.db"
Sep 22 23:22:26.977 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.977 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/011.replace" to "/tmp/downstairs-zrMnlo6G/00/000/011.completed"
Sep 22 23:22:26.977 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.977 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:26.978 DEBG [0] It's time to notify for 654
Sep 22 23:22:26.978 INFO Completion from [0] id:654 status:true
Sep 22 23:22:26.978 INFO [655/752] Repair commands completed
Sep 22 23:22:26.978 INFO Pop front: ReconcileIO { id: ReconciliationId(655), op: ExtentReopen { repair_id: ReconciliationId(655), extent_id: 17 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.978 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.978 INFO [0] received reconcile message
Sep 22 23:22:26.978 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(655), op: ExtentReopen { repair_id: ReconciliationId(655), extent_id: 17 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.978 INFO [0] client ExtentReopen { repair_id: ReconciliationId(655), extent_id: 17 }
Sep 22 23:22:26.978 INFO [1] received reconcile message
Sep 22 23:22:26.978 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(655), op: ExtentReopen { repair_id: ReconciliationId(655), extent_id: 17 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.978 INFO [1] client ExtentReopen { repair_id: ReconciliationId(655), extent_id: 17 }
Sep 22 23:22:26.978 INFO [2] received reconcile message
Sep 22 23:22:26.978 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(655), op: ExtentReopen { repair_id: ReconciliationId(655), extent_id: 17 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.978 INFO [2] client ExtentReopen { repair_id: ReconciliationId(655), extent_id: 17 }
Sep 22 23:22:26.978 DEBG 655 Reopen extent 17
Sep 22 23:22:26.979 DEBG 655 Reopen extent 17
Sep 22 23:22:26.979 DEBG 655 Reopen extent 17
Sep 22 23:22:26.980 DEBG [2] It's time to notify for 655
Sep 22 23:22:26.980 INFO Completion from [2] id:655 status:true
Sep 22 23:22:26.980 INFO [656/752] Repair commands completed
Sep 22 23:22:26.980 INFO Pop front: ReconcileIO { id: ReconciliationId(656), op: ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.980 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.980 INFO [0] received reconcile message
Sep 22 23:22:26.980 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(656), op: ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.980 INFO [0] client ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.980 INFO [1] received reconcile message
Sep 22 23:22:26.980 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(656), op: ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:26.980 INFO [1] client ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.980 INFO [2] received reconcile message
Sep 22 23:22:26.980 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(656), op: ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.980 INFO [2] client ExtentFlush { repair_id: ReconciliationId(656), extent_id: 116, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:26.980 DEBG 656 Flush extent 116 with f:2 g:2
Sep 22 23:22:26.980 DEBG Flush just extent 116 with f:2 and g:2
Sep 22 23:22:26.980 DEBG [1] It's time to notify for 656
Sep 22 23:22:26.980 INFO Completion from [1] id:656 status:true
Sep 22 23:22:26.980 INFO [657/752] Repair commands completed
Sep 22 23:22:26.980 INFO Pop front: ReconcileIO { id: ReconciliationId(657), op: ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.980 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.980 INFO [0] received reconcile message
Sep 22 23:22:26.981 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(657), op: ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.981 INFO [0] client ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }
Sep 22 23:22:26.981 INFO [1] received reconcile message
Sep 22 23:22:26.981 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(657), op: ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.981 INFO [1] client ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }
Sep 22 23:22:26.981 INFO [2] received reconcile message
Sep 22 23:22:26.981 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(657), op: ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:26.981 INFO [2] client ExtentClose { repair_id: ReconciliationId(657), extent_id: 116 }
Sep 22 23:22:26.981 DEBG 657 Close extent 116
Sep 22 23:22:26.981 DEBG 657 Close extent 116
Sep 22 23:22:26.981 DEBG 657 Close extent 116
Sep 22 23:22:26.982 DEBG [2] It's time to notify for 657
Sep 22 23:22:26.982 INFO Completion from [2] id:657 status:true
Sep 22 23:22:26.982 INFO [658/752] Repair commands completed
Sep 22 23:22:26.982 INFO Pop front: ReconcileIO { id: ReconciliationId(658), op: ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:26.982 INFO Sent repair work, now wait for resp
Sep 22 23:22:26.982 INFO [0] received reconcile message
Sep 22 23:22:26.982 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(658), op: ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:26.982 INFO [0] client ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.982 INFO [0] Sending repair request ReconciliationId(658)
Sep 22 23:22:26.982 INFO [1] received reconcile message
Sep 22 23:22:26.982 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(658), op: ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:26.982 INFO [1] client ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.982 INFO [1] No action required ReconciliationId(658)
Sep 22 23:22:26.982 INFO [2] received reconcile message
Sep 22 23:22:26.982 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(658), op: ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:26.982 INFO [2] client ExtentRepair { repair_id: ReconciliationId(658), extent_id: 116, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:26.982 INFO [2] No action required ReconciliationId(658)
Sep 22 23:22:26.982 DEBG 658 Repair extent 116 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:26.982 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/074.copy"
Sep 22 23:22:27.048 INFO accepted connection, remote_addr: 127.0.0.1:47179, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.048 TRCE incoming request, uri: /extent/116/files, method: GET, req_id: 9efeef61-d913-43a4-8513-2602c72583e8, remote_addr: 127.0.0.1:47179, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.048 INFO request completed, latency_us: 258, response_code: 200, uri: /extent/116/files, method: GET, req_id: 9efeef61-d913-43a4-8513-2602c72583e8, remote_addr: 127.0.0.1:47179, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.049 INFO eid:116 Found repair files: ["074", "074.db"]
Sep 22 23:22:27.049 TRCE incoming request, uri: /newextent/116/data, method: GET, req_id: 98559df6-11c1-4596-8101-3775f8c6373d, remote_addr: 127.0.0.1:47179, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.049 INFO request completed, latency_us: 361, response_code: 200, uri: /newextent/116/data, method: GET, req_id: 98559df6-11c1-4596-8101-3775f8c6373d, remote_addr: 127.0.0.1:47179, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.054 TRCE incoming request, uri: /newextent/116/db, method: GET, req_id: abfa9dac-0d0d-4f06-8a43-a11bb6ca438f, remote_addr: 127.0.0.1:47179, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.054 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/116/db, method: GET, req_id: abfa9dac-0d0d-4f06-8a43-a11bb6ca438f, remote_addr: 127.0.0.1:47179, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.055 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/074.copy" to "/tmp/downstairs-zrMnlo6G/00/000/074.replace"
Sep 22 23:22:27.056 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.056 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/074.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.057 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/074"
Sep 22 23:22:27.057 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/074.db"
Sep 22 23:22:27.057 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.057 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/074.replace" to "/tmp/downstairs-zrMnlo6G/00/000/074.completed"
Sep 22 23:22:27.057 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.057 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.057 DEBG [0] It's time to notify for 658
Sep 22 23:22:27.057 INFO Completion from [0] id:658 status:true
Sep 22 23:22:27.057 INFO [659/752] Repair commands completed
Sep 22 23:22:27.057 INFO Pop front: ReconcileIO { id: ReconciliationId(659), op: ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.057 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.057 INFO [0] received reconcile message
Sep 22 23:22:27.057 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(659), op: ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.057 INFO [0] client ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }
Sep 22 23:22:27.058 INFO [1] received reconcile message
Sep 22 23:22:27.058 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(659), op: ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.058 INFO [1] client ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }
Sep 22 23:22:27.058 INFO [2] received reconcile message
Sep 22 23:22:27.058 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(659), op: ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.058 INFO [2] client ExtentReopen { repair_id: ReconciliationId(659), extent_id: 116 }
Sep 22 23:22:27.058 DEBG 659 Reopen extent 116
Sep 22 23:22:27.058 DEBG 659 Reopen extent 116
Sep 22 23:22:27.059 DEBG 659 Reopen extent 116
Sep 22 23:22:27.060 DEBG [2] It's time to notify for 659
Sep 22 23:22:27.060 INFO Completion from [2] id:659 status:true
Sep 22 23:22:27.060 INFO [660/752] Repair commands completed
Sep 22 23:22:27.060 INFO Pop front: ReconcileIO { id: ReconciliationId(660), op: ExtentFlush { repair_id: ReconciliationId(660), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.060 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.060 INFO [0] received reconcile message
Sep 22 23:22:27.060 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(660), op: ExtentFlush { repair_id: ReconciliationId(660), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.060 INFO [0] client ExtentFlush { repair_id: ReconciliationId(660), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.060 INFO [1] received reconcile message
Sep 22 23:22:27.060 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(660), op: ExtentFlush { repair_id: ReconciliationId(660), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.060 INFO [1] client ExtentFlush { repair_id: ReconciliationId(660), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.060 INFO [2] received reconcile message
Sep 22 23:22:27.060 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(660), op: ExtentFlush { repair_id: ReconciliationId(660), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.060 INFO [2] client ExtentFlush { repair_id: ReconciliationId(660), extent_id: 142, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.060 DEBG 660 Flush extent 142 with f:2 g:2
Sep 22 23:22:27.060 DEBG Flush just extent 142 with f:2 and g:2
Sep 22 23:22:27.060 DEBG [1] It's time to notify for 660
Sep 22 23:22:27.060 INFO Completion from [1] id:660 status:true
Sep 22 23:22:27.060 INFO [661/752] Repair commands completed
Sep 22 23:22:27.060 INFO Pop front: ReconcileIO { id: ReconciliationId(661), op: ExtentClose { repair_id: ReconciliationId(661), extent_id: 142 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.060 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.060 INFO [0] received reconcile message
Sep 22 23:22:27.060 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(661), op: ExtentClose { repair_id: ReconciliationId(661), extent_id: 142 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.060 INFO [0] client ExtentClose { repair_id: ReconciliationId(661), extent_id: 142 }
Sep 22 23:22:27.060 INFO [1] received reconcile message
Sep 22 23:22:27.061 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(661), op: ExtentClose { repair_id: ReconciliationId(661), extent_id: 142 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.061 INFO [1] client ExtentClose { repair_id: ReconciliationId(661), extent_id: 142 }
Sep 22 23:22:27.061 INFO [2] received reconcile message
Sep 22 23:22:27.061 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(661), op: ExtentClose { repair_id: ReconciliationId(661), extent_id: 142 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.061 INFO [2] client ExtentClose { repair_id: ReconciliationId(661), extent_id: 142 }
Sep 22 23:22:27.061 DEBG 661 Close extent 142
Sep 22 23:22:27.061 DEBG 661 Close extent 142
Sep 22 23:22:27.061 DEBG 661 Close extent 142
Sep 22 23:22:27.062 DEBG [2] It's time to notify for 661
Sep 22 23:22:27.062 INFO Completion from [2] id:661 status:true
Sep 22 23:22:27.062 INFO [662/752] Repair commands completed
Sep 22 23:22:27.062 INFO Pop front: ReconcileIO { id: ReconciliationId(662), op: ExtentRepair { repair_id: ReconciliationId(662), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.062 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.062 INFO [0] received reconcile message
Sep 22 23:22:27.062 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(662), op: ExtentRepair { repair_id: ReconciliationId(662), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.062 INFO [0] client ExtentRepair { repair_id: ReconciliationId(662), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.062 INFO [0] Sending repair request ReconciliationId(662)
Sep 22 23:22:27.062 INFO [1] received reconcile message
Sep 22 23:22:27.062 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(662), op: ExtentRepair { repair_id: ReconciliationId(662), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.062 INFO [1] client ExtentRepair { repair_id: ReconciliationId(662), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.062 INFO [1] No action required ReconciliationId(662)
Sep 22 23:22:27.062 INFO [2] received reconcile message
Sep 22 23:22:27.062 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(662), op: ExtentRepair { repair_id: ReconciliationId(662), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.062 INFO [2] client ExtentRepair { repair_id: ReconciliationId(662), extent_id: 142, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.062 INFO [2] No action required ReconciliationId(662)
Sep 22 23:22:27.062 DEBG 662 Repair extent 142 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.062 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/08E.copy"
Sep 22 23:22:27.124 INFO accepted connection, remote_addr: 127.0.0.1:64380, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.124 TRCE incoming request, uri: /extent/142/files, method: GET, req_id: 8e1954a0-5ac9-4b2a-b556-473cf008d5c2, remote_addr: 127.0.0.1:64380, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.125 INFO request completed, latency_us: 209, response_code: 200, uri: /extent/142/files, method: GET, req_id: 8e1954a0-5ac9-4b2a-b556-473cf008d5c2, remote_addr: 127.0.0.1:64380, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.125 INFO eid:142 Found repair files: ["08E", "08E.db"]
Sep 22 23:22:27.125 TRCE incoming request, uri: /newextent/142/data, method: GET, req_id: 1f54fc79-308f-4a13-8be7-64c52c202998, remote_addr: 127.0.0.1:64380, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.125 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/142/data, method: GET, req_id: 1f54fc79-308f-4a13-8be7-64c52c202998, remote_addr: 127.0.0.1:64380, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.130 TRCE incoming request, uri: /newextent/142/db, method: GET, req_id: 1ad0bcd3-7925-47df-a263-97ef83220568, remote_addr: 127.0.0.1:64380, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.130 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/142/db, method: GET, req_id: 1ad0bcd3-7925-47df-a263-97ef83220568, remote_addr: 127.0.0.1:64380, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.131 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/08E.copy" to "/tmp/downstairs-zrMnlo6G/00/000/08E.replace"
Sep 22 23:22:27.131 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.132 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/08E.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.133 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08E"
Sep 22 23:22:27.133 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08E.db"
Sep 22 23:22:27.133 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.133 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/08E.replace" to "/tmp/downstairs-zrMnlo6G/00/000/08E.completed"
Sep 22 23:22:27.133 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.133 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.133 DEBG [0] It's time to notify for 662
Sep 22 23:22:27.133 INFO Completion from [0] id:662 status:true
Sep 22 23:22:27.133 INFO [663/752] Repair commands completed
Sep 22 23:22:27.133 INFO Pop front: ReconcileIO { id: ReconciliationId(663), op: ExtentReopen { repair_id: ReconciliationId(663), extent_id: 142 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.133 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.133 INFO [0] received reconcile message
Sep 22 23:22:27.133 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(663), op: ExtentReopen { repair_id: ReconciliationId(663), extent_id: 142 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.133 INFO [0] client ExtentReopen { repair_id: ReconciliationId(663), extent_id: 142 }
Sep 22 23:22:27.133 INFO [1] received reconcile message
Sep 22 23:22:27.133 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(663), op: ExtentReopen { repair_id: ReconciliationId(663), extent_id: 142 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.133 INFO [1] client ExtentReopen { repair_id: ReconciliationId(663), extent_id: 142 }
Sep 22 23:22:27.133 INFO [2] received reconcile message
Sep 22 23:22:27.133 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(663), op: ExtentReopen { repair_id: ReconciliationId(663), extent_id: 142 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.133 INFO [2] client ExtentReopen { repair_id: ReconciliationId(663), extent_id: 142 }
Sep 22 23:22:27.134 DEBG 663 Reopen extent 142
Sep 22 23:22:27.134 DEBG 663 Reopen extent 142
Sep 22 23:22:27.135 DEBG 663 Reopen extent 142
Sep 22 23:22:27.135 DEBG [2] It's time to notify for 663
Sep 22 23:22:27.135 INFO Completion from [2] id:663 status:true
Sep 22 23:22:27.135 INFO [664/752] Repair commands completed
Sep 22 23:22:27.135 INFO Pop front: ReconcileIO { id: ReconciliationId(664), op: ExtentFlush { repair_id: ReconciliationId(664), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.135 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.135 INFO [0] received reconcile message
Sep 22 23:22:27.135 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(664), op: ExtentFlush { repair_id: ReconciliationId(664), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.135 INFO [0] client ExtentFlush { repair_id: ReconciliationId(664), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.135 INFO [1] received reconcile message
Sep 22 23:22:27.135 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(664), op: ExtentFlush { repair_id: ReconciliationId(664), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.135 INFO [1] client ExtentFlush { repair_id: ReconciliationId(664), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.136 INFO [2] received reconcile message
Sep 22 23:22:27.136 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(664), op: ExtentFlush { repair_id: ReconciliationId(664), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.136 INFO [2] client ExtentFlush { repair_id: ReconciliationId(664), extent_id: 88, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.136 DEBG 664 Flush extent 88 with f:2 g:2
Sep 22 23:22:27.136 DEBG Flush just extent 88 with f:2 and g:2
Sep 22 23:22:27.136 DEBG [1] It's time to notify for 664
Sep 22 23:22:27.136 INFO Completion from [1] id:664 status:true
Sep 22 23:22:27.136 INFO [665/752] Repair commands completed
Sep 22 23:22:27.136 INFO Pop front: ReconcileIO { id: ReconciliationId(665), op: ExtentClose { repair_id: ReconciliationId(665), extent_id: 88 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.136 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.136 INFO [0] received reconcile message
Sep 22 23:22:27.136 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(665), op: ExtentClose { repair_id: ReconciliationId(665), extent_id: 88 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.136 INFO [0] client ExtentClose { repair_id: ReconciliationId(665), extent_id: 88 }
Sep 22 23:22:27.136 INFO [1] received reconcile message
Sep 22 23:22:27.136 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(665), op: ExtentClose { repair_id: ReconciliationId(665), extent_id: 88 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.136 INFO [1] client ExtentClose { repair_id: ReconciliationId(665), extent_id: 88 }
Sep 22 23:22:27.136 INFO [2] received reconcile message
Sep 22 23:22:27.136 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(665), op: ExtentClose { repair_id: ReconciliationId(665), extent_id: 88 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.136 INFO [2] client ExtentClose { repair_id: ReconciliationId(665), extent_id: 88 }
Sep 22 23:22:27.136 DEBG 665 Close extent 88
Sep 22 23:22:27.137 DEBG 665 Close extent 88
Sep 22 23:22:27.137 DEBG 665 Close extent 88
Sep 22 23:22:27.137 DEBG [2] It's time to notify for 665
Sep 22 23:22:27.137 INFO Completion from [2] id:665 status:true
Sep 22 23:22:27.137 INFO [666/752] Repair commands completed
Sep 22 23:22:27.137 INFO Pop front: ReconcileIO { id: ReconciliationId(666), op: ExtentRepair { repair_id: ReconciliationId(666), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.137 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.137 INFO [0] received reconcile message
Sep 22 23:22:27.137 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(666), op: ExtentRepair { repair_id: ReconciliationId(666), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.137 INFO [0] client ExtentRepair { repair_id: ReconciliationId(666), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.137 INFO [0] Sending repair request ReconciliationId(666)
Sep 22 23:22:27.138 INFO [1] received reconcile message
Sep 22 23:22:27.138 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(666), op: ExtentRepair { repair_id: ReconciliationId(666), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.138 INFO [1] client ExtentRepair { repair_id: ReconciliationId(666), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.138 INFO [1] No action required ReconciliationId(666)
Sep 22 23:22:27.138 INFO [2] received reconcile message
Sep 22 23:22:27.138 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(666), op: ExtentRepair { repair_id: ReconciliationId(666), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.138 INFO [2] client ExtentRepair { repair_id: ReconciliationId(666), extent_id: 88, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.138 INFO [2] No action required ReconciliationId(666)
Sep 22 23:22:27.138 DEBG 666 Repair extent 88 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.138 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/058.copy"
Sep 22 23:22:27.203 INFO accepted connection, remote_addr: 127.0.0.1:58448, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.204 TRCE incoming request, uri: /extent/88/files, method: GET, req_id: c407c640-7177-486a-8c62-36c8a040d987, remote_addr: 127.0.0.1:58448, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.204 INFO request completed, latency_us: 201, response_code: 200, uri: /extent/88/files, method: GET, req_id: c407c640-7177-486a-8c62-36c8a040d987, remote_addr: 127.0.0.1:58448, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.204 INFO eid:88 Found repair files: ["058", "058.db"]
Sep 22 23:22:27.204 TRCE incoming request, uri: /newextent/88/data, method: GET, req_id: c9c98208-e646-4784-9b69-9a0951e9ef6d, remote_addr: 127.0.0.1:58448, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.205 INFO request completed, latency_us: 324, response_code: 200, uri: /newextent/88/data, method: GET, req_id: c9c98208-e646-4784-9b69-9a0951e9ef6d, remote_addr: 127.0.0.1:58448, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.209 TRCE incoming request, uri: /newextent/88/db, method: GET, req_id: 83019044-db17-471f-95a7-3a9aca6fa93f, remote_addr: 127.0.0.1:58448, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.210 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/88/db, method: GET, req_id: 83019044-db17-471f-95a7-3a9aca6fa93f, remote_addr: 127.0.0.1:58448, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.211 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/058.copy" to "/tmp/downstairs-zrMnlo6G/00/000/058.replace"
Sep 22 23:22:27.211 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.212 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/058.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.212 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/058"
Sep 22 23:22:27.212 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/058.db"
Sep 22 23:22:27.212 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.212 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/058.replace" to "/tmp/downstairs-zrMnlo6G/00/000/058.completed"
Sep 22 23:22:27.212 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.212 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.212 DEBG [0] It's time to notify for 666
Sep 22 23:22:27.213 INFO Completion from [0] id:666 status:true
Sep 22 23:22:27.213 INFO [667/752] Repair commands completed
Sep 22 23:22:27.213 INFO Pop front: ReconcileIO { id: ReconciliationId(667), op: ExtentReopen { repair_id: ReconciliationId(667), extent_id: 88 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.213 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.213 INFO [0] received reconcile message
Sep 22 23:22:27.213 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(667), op: ExtentReopen { repair_id: ReconciliationId(667), extent_id: 88 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.213 INFO [0] client ExtentReopen { repair_id: ReconciliationId(667), extent_id: 88 }
Sep 22 23:22:27.213 INFO [1] received reconcile message
Sep 22 23:22:27.213 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(667), op: ExtentReopen { repair_id: ReconciliationId(667), extent_id: 88 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.213 INFO [1] client ExtentReopen { repair_id: ReconciliationId(667), extent_id: 88 }
Sep 22 23:22:27.213 INFO [2] received reconcile message
Sep 22 23:22:27.213 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(667), op: ExtentReopen { repair_id: ReconciliationId(667), extent_id: 88 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.213 INFO [2] client ExtentReopen { repair_id: ReconciliationId(667), extent_id: 88 }
Sep 22 23:22:27.213 DEBG 667 Reopen extent 88
Sep 22 23:22:27.214 DEBG 667 Reopen extent 88
Sep 22 23:22:27.214 DEBG 667 Reopen extent 88
Sep 22 23:22:27.215 DEBG [2] It's time to notify for 667
Sep 22 23:22:27.215 INFO Completion from [2] id:667 status:true
Sep 22 23:22:27.215 INFO [668/752] Repair commands completed
Sep 22 23:22:27.215 INFO Pop front: ReconcileIO { id: ReconciliationId(668), op: ExtentFlush { repair_id: ReconciliationId(668), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.215 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.215 INFO [0] received reconcile message
Sep 22 23:22:27.215 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(668), op: ExtentFlush { repair_id: ReconciliationId(668), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.215 INFO [0] client ExtentFlush { repair_id: ReconciliationId(668), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.215 INFO [1] received reconcile message
Sep 22 23:22:27.215 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(668), op: ExtentFlush { repair_id: ReconciliationId(668), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.215 INFO [1] client ExtentFlush { repair_id: ReconciliationId(668), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.215 INFO [2] received reconcile message
Sep 22 23:22:27.215 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(668), op: ExtentFlush { repair_id: ReconciliationId(668), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.215 INFO [2] client ExtentFlush { repair_id: ReconciliationId(668), extent_id: 51, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.215 DEBG 668 Flush extent 51 with f:2 g:2
Sep 22 23:22:27.215 DEBG Flush just extent 51 with f:2 and g:2
Sep 22 23:22:27.215 DEBG [1] It's time to notify for 668
Sep 22 23:22:27.215 INFO Completion from [1] id:668 status:true
Sep 22 23:22:27.215 INFO [669/752] Repair commands completed
Sep 22 23:22:27.215 INFO Pop front: ReconcileIO { id: ReconciliationId(669), op: ExtentClose { repair_id: ReconciliationId(669), extent_id: 51 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.215 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.215 INFO [0] received reconcile message
Sep 22 23:22:27.215 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(669), op: ExtentClose { repair_id: ReconciliationId(669), extent_id: 51 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.215 INFO [0] client ExtentClose { repair_id: ReconciliationId(669), extent_id: 51 }
Sep 22 23:22:27.215 INFO [1] received reconcile message
Sep 22 23:22:27.215 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(669), op: ExtentClose { repair_id: ReconciliationId(669), extent_id: 51 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.215 INFO [1] client ExtentClose { repair_id: ReconciliationId(669), extent_id: 51 }
Sep 22 23:22:27.216 INFO [2] received reconcile message
Sep 22 23:22:27.216 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(669), op: ExtentClose { repair_id: ReconciliationId(669), extent_id: 51 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.216 INFO [2] client ExtentClose { repair_id: ReconciliationId(669), extent_id: 51 }
Sep 22 23:22:27.216 DEBG 669 Close extent 51
Sep 22 23:22:27.216 DEBG 669 Close extent 51
Sep 22 23:22:27.216 DEBG 669 Close extent 51
Sep 22 23:22:27.217 DEBG [2] It's time to notify for 669
Sep 22 23:22:27.217 INFO Completion from [2] id:669 status:true
Sep 22 23:22:27.217 INFO [670/752] Repair commands completed
Sep 22 23:22:27.217 INFO Pop front: ReconcileIO { id: ReconciliationId(670), op: ExtentRepair { repair_id: ReconciliationId(670), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.217 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.217 INFO [0] received reconcile message
Sep 22 23:22:27.217 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(670), op: ExtentRepair { repair_id: ReconciliationId(670), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.217 INFO [0] client ExtentRepair { repair_id: ReconciliationId(670), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.217 INFO [0] Sending repair request ReconciliationId(670)
Sep 22 23:22:27.217 INFO [1] received reconcile message
Sep 22 23:22:27.217 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(670), op: ExtentRepair { repair_id: ReconciliationId(670), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.217 INFO [1] client ExtentRepair { repair_id: ReconciliationId(670), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.217 INFO [1] No action required ReconciliationId(670)
Sep 22 23:22:27.217 INFO [2] received reconcile message
Sep 22 23:22:27.217 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(670), op: ExtentRepair { repair_id: ReconciliationId(670), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.217 INFO [2] client ExtentRepair { repair_id: ReconciliationId(670), extent_id: 51, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.217 INFO [2] No action required ReconciliationId(670)
Sep 22 23:22:27.217 DEBG 670 Repair extent 51 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.217 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/033.copy"
Sep 22 23:22:27.283 INFO accepted connection, remote_addr: 127.0.0.1:59452, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.283 TRCE incoming request, uri: /extent/51/files, method: GET, req_id: 3559e9f7-d34a-4c0d-af86-4d20e1cd1f89, remote_addr: 127.0.0.1:59452, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.283 INFO request completed, latency_us: 201, response_code: 200, uri: /extent/51/files, method: GET, req_id: 3559e9f7-d34a-4c0d-af86-4d20e1cd1f89, remote_addr: 127.0.0.1:59452, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.283 INFO eid:51 Found repair files: ["033", "033.db"]
Sep 22 23:22:27.284 TRCE incoming request, uri: /newextent/51/data, method: GET, req_id: fe390388-1975-4ea7-99f4-bad030ca68d2, remote_addr: 127.0.0.1:59452, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.284 INFO request completed, latency_us: 260, response_code: 200, uri: /newextent/51/data, method: GET, req_id: fe390388-1975-4ea7-99f4-bad030ca68d2, remote_addr: 127.0.0.1:59452, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.289 TRCE incoming request, uri: /newextent/51/db, method: GET, req_id: 9f2bb5d5-f947-4c96-b29e-f0af846d5b60, remote_addr: 127.0.0.1:59452, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.289 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/51/db, method: GET, req_id: 9f2bb5d5-f947-4c96-b29e-f0af846d5b60, remote_addr: 127.0.0.1:59452, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.290 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/033.copy" to "/tmp/downstairs-zrMnlo6G/00/000/033.replace"
Sep 22 23:22:27.290 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.291 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/033.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.291 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/033"
Sep 22 23:22:27.291 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/033.db"
Sep 22 23:22:27.291 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.291 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/033.replace" to "/tmp/downstairs-zrMnlo6G/00/000/033.completed"
Sep 22 23:22:27.291 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.291 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.291 DEBG [0] It's time to notify for 670
Sep 22 23:22:27.292 INFO Completion from [0] id:670 status:true
Sep 22 23:22:27.292 INFO [671/752] Repair commands completed
Sep 22 23:22:27.292 INFO Pop front: ReconcileIO { id: ReconciliationId(671), op: ExtentReopen { repair_id: ReconciliationId(671), extent_id: 51 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.292 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.292 INFO [0] received reconcile message
Sep 22 23:22:27.292 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(671), op: ExtentReopen { repair_id: ReconciliationId(671), extent_id: 51 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.292 INFO [0] client ExtentReopen { repair_id: ReconciliationId(671), extent_id: 51 }
Sep 22 23:22:27.292 INFO [1] received reconcile message
Sep 22 23:22:27.292 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(671), op: ExtentReopen { repair_id: ReconciliationId(671), extent_id: 51 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.292 INFO [1] client ExtentReopen { repair_id: ReconciliationId(671), extent_id: 51 }
Sep 22 23:22:27.292 INFO [2] received reconcile message
Sep 22 23:22:27.292 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(671), op: ExtentReopen { repair_id: ReconciliationId(671), extent_id: 51 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.292 INFO [2] client ExtentReopen { repair_id: ReconciliationId(671), extent_id: 51 }
Sep 22 23:22:27.292 DEBG 671 Reopen extent 51
Sep 22 23:22:27.293 DEBG 671 Reopen extent 51
Sep 22 23:22:27.293 DEBG 671 Reopen extent 51
Sep 22 23:22:27.294 DEBG [2] It's time to notify for 671
Sep 22 23:22:27.294 INFO Completion from [2] id:671 status:true
Sep 22 23:22:27.294 INFO [672/752] Repair commands completed
Sep 22 23:22:27.294 INFO Pop front: ReconcileIO { id: ReconciliationId(672), op: ExtentFlush { repair_id: ReconciliationId(672), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.294 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.294 INFO [0] received reconcile message
Sep 22 23:22:27.294 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(672), op: ExtentFlush { repair_id: ReconciliationId(672), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.294 INFO [0] client ExtentFlush { repair_id: ReconciliationId(672), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.294 INFO [1] received reconcile message
Sep 22 23:22:27.294 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(672), op: ExtentFlush { repair_id: ReconciliationId(672), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.294 INFO [1] client ExtentFlush { repair_id: ReconciliationId(672), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.294 INFO [2] received reconcile message
Sep 22 23:22:27.294 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(672), op: ExtentFlush { repair_id: ReconciliationId(672), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.294 INFO [2] client ExtentFlush { repair_id: ReconciliationId(672), extent_id: 33, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.294 DEBG 672 Flush extent 33 with f:2 g:2
Sep 22 23:22:27.294 DEBG Flush just extent 33 with f:2 and g:2
Sep 22 23:22:27.294 DEBG [1] It's time to notify for 672
Sep 22 23:22:27.294 INFO Completion from [1] id:672 status:true
Sep 22 23:22:27.294 INFO [673/752] Repair commands completed
Sep 22 23:22:27.294 INFO Pop front: ReconcileIO { id: ReconciliationId(673), op: ExtentClose { repair_id: ReconciliationId(673), extent_id: 33 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.294 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.294 INFO [0] received reconcile message
Sep 22 23:22:27.295 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(673), op: ExtentClose { repair_id: ReconciliationId(673), extent_id: 33 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.295 INFO [0] client ExtentClose { repair_id: ReconciliationId(673), extent_id: 33 }
Sep 22 23:22:27.295 INFO [1] received reconcile message
Sep 22 23:22:27.295 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(673), op: ExtentClose { repair_id: ReconciliationId(673), extent_id: 33 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.295 INFO [1] client ExtentClose { repair_id: ReconciliationId(673), extent_id: 33 }
Sep 22 23:22:27.295 INFO [2] received reconcile message
Sep 22 23:22:27.295 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(673), op: ExtentClose { repair_id: ReconciliationId(673), extent_id: 33 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.295 INFO [2] client ExtentClose { repair_id: ReconciliationId(673), extent_id: 33 }
Sep 22 23:22:27.295 DEBG 673 Close extent 33
Sep 22 23:22:27.295 DEBG 673 Close extent 33
Sep 22 23:22:27.295 DEBG 673 Close extent 33
Sep 22 23:22:27.296 DEBG [2] It's time to notify for 673
Sep 22 23:22:27.296 INFO Completion from [2] id:673 status:true
Sep 22 23:22:27.296 INFO [674/752] Repair commands completed
Sep 22 23:22:27.296 INFO Pop front: ReconcileIO { id: ReconciliationId(674), op: ExtentRepair { repair_id: ReconciliationId(674), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.296 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.296 INFO [0] received reconcile message
Sep 22 23:22:27.296 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(674), op: ExtentRepair { repair_id: ReconciliationId(674), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.296 INFO [0] client ExtentRepair { repair_id: ReconciliationId(674), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.296 INFO [0] Sending repair request ReconciliationId(674)
Sep 22 23:22:27.296 INFO [1] received reconcile message
Sep 22 23:22:27.296 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(674), op: ExtentRepair { repair_id: ReconciliationId(674), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.296 INFO [1] client ExtentRepair { repair_id: ReconciliationId(674), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.296 INFO [1] No action required ReconciliationId(674)
Sep 22 23:22:27.296 INFO [2] received reconcile message
Sep 22 23:22:27.296 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(674), op: ExtentRepair { repair_id: ReconciliationId(674), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.296 INFO [2] client ExtentRepair { repair_id: ReconciliationId(674), extent_id: 33, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.296 INFO [2] No action required ReconciliationId(674)
Sep 22 23:22:27.296 DEBG 674 Repair extent 33 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.296 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/021.copy"
Sep 22 23:22:27.331 DEBG [rc] retire 1090 clears [JobId(1089), JobId(1090)], : downstairs
Sep 22 23:22:27.331 DEBG IO Flush 1094 has deps [JobId(1093), JobId(1092)]
Sep 22 23:22:27.331 DEBG Flush     :1092 extent_limit None deps:[JobId(1091), JobId(1090)] res:true f:35 g:1
Sep 22 23:22:27.331 WARN returning error on read!
Sep 22 23:22:27.331 DEBG Read      :1093 deps:[JobId(1092)] res:false
Sep 22 23:22:27.331 INFO [lossy] skipping 1093
Sep 22 23:22:27.337 DEBG Read      :1093 deps:[JobId(1092)] res:true
Sep 22 23:22:27.359 ERRO [0] job id 1093 saw error GenericError("test error")
Sep 22 23:22:27.361 INFO accepted connection, remote_addr: 127.0.0.1:59101, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.361 DEBG Flush     :1094 extent_limit None deps:[JobId(1093), JobId(1092)] res:true f:36 g:1
Sep 22 23:22:27.361 INFO [lossy] sleeping 1 second
Sep 22 23:22:27.361 TRCE incoming request, uri: /extent/33/files, method: GET, req_id: 5d4e7ced-e28b-42e5-9cea-2717ee73cfc2, remote_addr: 127.0.0.1:59101, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.362 INFO request completed, latency_us: 213, response_code: 200, uri: /extent/33/files, method: GET, req_id: 5d4e7ced-e28b-42e5-9cea-2717ee73cfc2, remote_addr: 127.0.0.1:59101, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.362 INFO eid:33 Found repair files: ["021", "021.db"]
Sep 22 23:22:27.362 TRCE incoming request, uri: /newextent/33/data, method: GET, req_id: 8ce52ddf-8d30-4839-9a02-90455e86700f, remote_addr: 127.0.0.1:59101, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.362 INFO request completed, latency_us: 316, response_code: 200, uri: /newextent/33/data, method: GET, req_id: 8ce52ddf-8d30-4839-9a02-90455e86700f, remote_addr: 127.0.0.1:59101, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.363 DEBG Flush     :1092 extent_limit None deps:[JobId(1091), JobId(1090)] res:true f:35 g:1
Sep 22 23:22:27.367 TRCE incoming request, uri: /newextent/33/db, method: GET, req_id: 501ab7ce-34e9-4ea3-b6cd-4ff218682319, remote_addr: 127.0.0.1:59101, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.367 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/33/db, method: GET, req_id: 501ab7ce-34e9-4ea3-b6cd-4ff218682319, remote_addr: 127.0.0.1:59101, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.369 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/021.copy" to "/tmp/downstairs-zrMnlo6G/00/000/021.replace"
Sep 22 23:22:27.369 DEBG Read      :1093 deps:[JobId(1092)] res:true
Sep 22 23:22:27.369 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.369 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/021.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.370 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/021"
Sep 22 23:22:27.370 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/021.db"
Sep 22 23:22:27.370 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.370 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/021.replace" to "/tmp/downstairs-zrMnlo6G/00/000/021.completed"
Sep 22 23:22:27.370 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.370 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.370 DEBG [0] It's time to notify for 674
Sep 22 23:22:27.370 INFO Completion from [0] id:674 status:true
Sep 22 23:22:27.370 INFO [675/752] Repair commands completed
Sep 22 23:22:27.370 INFO Pop front: ReconcileIO { id: ReconciliationId(675), op: ExtentReopen { repair_id: ReconciliationId(675), extent_id: 33 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.370 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.370 INFO [0] received reconcile message
Sep 22 23:22:27.370 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(675), op: ExtentReopen { repair_id: ReconciliationId(675), extent_id: 33 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.370 INFO [0] client ExtentReopen { repair_id: ReconciliationId(675), extent_id: 33 }
Sep 22 23:22:27.370 INFO [1] received reconcile message
Sep 22 23:22:27.371 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(675), op: ExtentReopen { repair_id: ReconciliationId(675), extent_id: 33 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.371 INFO [1] client ExtentReopen { repair_id: ReconciliationId(675), extent_id: 33 }
Sep 22 23:22:27.371 INFO [2] received reconcile message
Sep 22 23:22:27.371 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(675), op: ExtentReopen { repair_id: ReconciliationId(675), extent_id: 33 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.371 INFO [2] client ExtentReopen { repair_id: ReconciliationId(675), extent_id: 33 }
Sep 22 23:22:27.371 DEBG 675 Reopen extent 33
Sep 22 23:22:27.371 DEBG 675 Reopen extent 33
Sep 22 23:22:27.372 DEBG 675 Reopen extent 33
Sep 22 23:22:27.372 DEBG [2] It's time to notify for 675
Sep 22 23:22:27.373 INFO Completion from [2] id:675 status:true
Sep 22 23:22:27.373 INFO [676/752] Repair commands completed
Sep 22 23:22:27.373 INFO Pop front: ReconcileIO { id: ReconciliationId(676), op: ExtentFlush { repair_id: ReconciliationId(676), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.373 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.373 INFO [0] received reconcile message
Sep 22 23:22:27.373 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(676), op: ExtentFlush { repair_id: ReconciliationId(676), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.373 INFO [0] client ExtentFlush { repair_id: ReconciliationId(676), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.373 INFO [1] received reconcile message
Sep 22 23:22:27.373 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(676), op: ExtentFlush { repair_id: ReconciliationId(676), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.373 INFO [1] client ExtentFlush { repair_id: ReconciliationId(676), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.373 INFO [2] received reconcile message
Sep 22 23:22:27.373 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(676), op: ExtentFlush { repair_id: ReconciliationId(676), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.373 INFO [2] client ExtentFlush { repair_id: ReconciliationId(676), extent_id: 54, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.373 DEBG 676 Flush extent 54 with f:2 g:2
Sep 22 23:22:27.373 DEBG Flush just extent 54 with f:2 and g:2
Sep 22 23:22:27.373 DEBG [1] It's time to notify for 676
Sep 22 23:22:27.373 INFO Completion from [1] id:676 status:true
Sep 22 23:22:27.373 INFO [677/752] Repair commands completed
Sep 22 23:22:27.373 INFO Pop front: ReconcileIO { id: ReconciliationId(677), op: ExtentClose { repair_id: ReconciliationId(677), extent_id: 54 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.373 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.373 INFO [0] received reconcile message
Sep 22 23:22:27.373 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(677), op: ExtentClose { repair_id: ReconciliationId(677), extent_id: 54 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.373 INFO [0] client ExtentClose { repair_id: ReconciliationId(677), extent_id: 54 }
Sep 22 23:22:27.373 INFO [1] received reconcile message
Sep 22 23:22:27.373 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(677), op: ExtentClose { repair_id: ReconciliationId(677), extent_id: 54 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.373 INFO [1] client ExtentClose { repair_id: ReconciliationId(677), extent_id: 54 }
Sep 22 23:22:27.373 INFO [2] received reconcile message
Sep 22 23:22:27.373 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(677), op: ExtentClose { repair_id: ReconciliationId(677), extent_id: 54 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.373 INFO [2] client ExtentClose { repair_id: ReconciliationId(677), extent_id: 54 }
Sep 22 23:22:27.374 DEBG 677 Close extent 54
Sep 22 23:22:27.374 DEBG 677 Close extent 54
Sep 22 23:22:27.374 DEBG 677 Close extent 54
Sep 22 23:22:27.374 DEBG [2] It's time to notify for 677
Sep 22 23:22:27.375 INFO Completion from [2] id:677 status:true
Sep 22 23:22:27.375 INFO [678/752] Repair commands completed
Sep 22 23:22:27.375 INFO Pop front: ReconcileIO { id: ReconciliationId(678), op: ExtentRepair { repair_id: ReconciliationId(678), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.375 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.375 INFO [0] received reconcile message
Sep 22 23:22:27.375 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(678), op: ExtentRepair { repair_id: ReconciliationId(678), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.375 INFO [0] client ExtentRepair { repair_id: ReconciliationId(678), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.375 INFO [0] Sending repair request ReconciliationId(678)
Sep 22 23:22:27.375 INFO [1] received reconcile message
Sep 22 23:22:27.375 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(678), op: ExtentRepair { repair_id: ReconciliationId(678), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.375 INFO [1] client ExtentRepair { repair_id: ReconciliationId(678), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.375 INFO [1] No action required ReconciliationId(678)
Sep 22 23:22:27.375 INFO [2] received reconcile message
Sep 22 23:22:27.375 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(678), op: ExtentRepair { repair_id: ReconciliationId(678), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.375 INFO [2] client ExtentRepair { repair_id: ReconciliationId(678), extent_id: 54, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.375 INFO [2] No action required ReconciliationId(678)
Sep 22 23:22:27.375 DEBG 678 Repair extent 54 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.375 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/036.copy"
Sep 22 23:22:27.438 INFO accepted connection, remote_addr: 127.0.0.1:59164, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.438 TRCE incoming request, uri: /extent/54/files, method: GET, req_id: 044f231c-1ff5-452e-bc67-2009ae647edb, remote_addr: 127.0.0.1:59164, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.438 INFO request completed, latency_us: 199, response_code: 200, uri: /extent/54/files, method: GET, req_id: 044f231c-1ff5-452e-bc67-2009ae647edb, remote_addr: 127.0.0.1:59164, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.438 INFO eid:54 Found repair files: ["036", "036.db"]
Sep 22 23:22:27.439 TRCE incoming request, uri: /newextent/54/data, method: GET, req_id: 62311f09-c851-48fd-9d5e-02a587aec6c1, remote_addr: 127.0.0.1:59164, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.439 INFO request completed, latency_us: 260, response_code: 200, uri: /newextent/54/data, method: GET, req_id: 62311f09-c851-48fd-9d5e-02a587aec6c1, remote_addr: 127.0.0.1:59164, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.444 TRCE incoming request, uri: /newextent/54/db, method: GET, req_id: 9e0d4a93-7ac7-4def-8ca7-385b316decf2, remote_addr: 127.0.0.1:59164, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.444 INFO request completed, latency_us: 299, response_code: 200, uri: /newextent/54/db, method: GET, req_id: 9e0d4a93-7ac7-4def-8ca7-385b316decf2, remote_addr: 127.0.0.1:59164, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.445 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/036.copy" to "/tmp/downstairs-zrMnlo6G/00/000/036.replace"
Sep 22 23:22:27.445 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.446 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/036.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.446 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/036"
Sep 22 23:22:27.446 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/036.db"
Sep 22 23:22:27.446 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.446 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/036.replace" to "/tmp/downstairs-zrMnlo6G/00/000/036.completed"
Sep 22 23:22:27.446 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.446 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.447 DEBG [0] It's time to notify for 678
Sep 22 23:22:27.447 INFO Completion from [0] id:678 status:true
Sep 22 23:22:27.447 INFO [679/752] Repair commands completed
Sep 22 23:22:27.447 INFO Pop front: ReconcileIO { id: ReconciliationId(679), op: ExtentReopen { repair_id: ReconciliationId(679), extent_id: 54 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.447 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.447 INFO [0] received reconcile message
Sep 22 23:22:27.447 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(679), op: ExtentReopen { repair_id: ReconciliationId(679), extent_id: 54 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.447 INFO [0] client ExtentReopen { repair_id: ReconciliationId(679), extent_id: 54 }
Sep 22 23:22:27.447 INFO [1] received reconcile message
Sep 22 23:22:27.447 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(679), op: ExtentReopen { repair_id: ReconciliationId(679), extent_id: 54 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.447 INFO [1] client ExtentReopen { repair_id: ReconciliationId(679), extent_id: 54 }
Sep 22 23:22:27.447 INFO [2] received reconcile message
Sep 22 23:22:27.447 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(679), op: ExtentReopen { repair_id: ReconciliationId(679), extent_id: 54 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.447 INFO [2] client ExtentReopen { repair_id: ReconciliationId(679), extent_id: 54 }
Sep 22 23:22:27.447 DEBG 679 Reopen extent 54
Sep 22 23:22:27.448 DEBG 679 Reopen extent 54
Sep 22 23:22:27.448 DEBG 679 Reopen extent 54
Sep 22 23:22:27.449 DEBG [2] It's time to notify for 679
Sep 22 23:22:27.449 INFO Completion from [2] id:679 status:true
Sep 22 23:22:27.449 INFO [680/752] Repair commands completed
Sep 22 23:22:27.449 INFO Pop front: ReconcileIO { id: ReconciliationId(680), op: ExtentFlush { repair_id: ReconciliationId(680), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.449 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.449 INFO [0] received reconcile message
Sep 22 23:22:27.449 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(680), op: ExtentFlush { repair_id: ReconciliationId(680), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.449 INFO [0] client ExtentFlush { repair_id: ReconciliationId(680), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.449 INFO [1] received reconcile message
Sep 22 23:22:27.449 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(680), op: ExtentFlush { repair_id: ReconciliationId(680), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.449 INFO [1] client ExtentFlush { repair_id: ReconciliationId(680), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.449 INFO [2] received reconcile message
Sep 22 23:22:27.449 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(680), op: ExtentFlush { repair_id: ReconciliationId(680), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.449 INFO [2] client ExtentFlush { repair_id: ReconciliationId(680), extent_id: 112, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.449 DEBG 680 Flush extent 112 with f:2 g:2
Sep 22 23:22:27.449 DEBG Flush just extent 112 with f:2 and g:2
Sep 22 23:22:27.449 DEBG [1] It's time to notify for 680
Sep 22 23:22:27.449 INFO Completion from [1] id:680 status:true
Sep 22 23:22:27.449 INFO [681/752] Repair commands completed
Sep 22 23:22:27.449 INFO Pop front: ReconcileIO { id: ReconciliationId(681), op: ExtentClose { repair_id: ReconciliationId(681), extent_id: 112 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.449 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.450 INFO [0] received reconcile message
Sep 22 23:22:27.450 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(681), op: ExtentClose { repair_id: ReconciliationId(681), extent_id: 112 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.450 INFO [0] client ExtentClose { repair_id: ReconciliationId(681), extent_id: 112 }
Sep 22 23:22:27.450 INFO [1] received reconcile message
Sep 22 23:22:27.450 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(681), op: ExtentClose { repair_id: ReconciliationId(681), extent_id: 112 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.450 INFO [1] client ExtentClose { repair_id: ReconciliationId(681), extent_id: 112 }
Sep 22 23:22:27.450 INFO [2] received reconcile message
Sep 22 23:22:27.450 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(681), op: ExtentClose { repair_id: ReconciliationId(681), extent_id: 112 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.450 INFO [2] client ExtentClose { repair_id: ReconciliationId(681), extent_id: 112 }
Sep 22 23:22:27.450 DEBG 681 Close extent 112
Sep 22 23:22:27.450 DEBG 681 Close extent 112
Sep 22 23:22:27.450 DEBG 681 Close extent 112
Sep 22 23:22:27.451 DEBG [2] It's time to notify for 681
Sep 22 23:22:27.451 INFO Completion from [2] id:681 status:true
Sep 22 23:22:27.451 INFO [682/752] Repair commands completed
Sep 22 23:22:27.451 INFO Pop front: ReconcileIO { id: ReconciliationId(682), op: ExtentRepair { repair_id: ReconciliationId(682), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.451 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.451 INFO [0] received reconcile message
Sep 22 23:22:27.451 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(682), op: ExtentRepair { repair_id: ReconciliationId(682), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.451 INFO [0] client ExtentRepair { repair_id: ReconciliationId(682), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.451 INFO [0] Sending repair request ReconciliationId(682)
Sep 22 23:22:27.451 INFO [1] received reconcile message
Sep 22 23:22:27.451 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(682), op: ExtentRepair { repair_id: ReconciliationId(682), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.451 INFO [1] client ExtentRepair { repair_id: ReconciliationId(682), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.451 INFO [1] No action required ReconciliationId(682)
Sep 22 23:22:27.451 INFO [2] received reconcile message
Sep 22 23:22:27.451 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(682), op: ExtentRepair { repair_id: ReconciliationId(682), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.451 INFO [2] client ExtentRepair { repair_id: ReconciliationId(682), extent_id: 112, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.451 INFO [2] No action required ReconciliationId(682)
Sep 22 23:22:27.451 DEBG 682 Repair extent 112 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.451 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/070.copy"
Sep 22 23:22:27.512 INFO accepted connection, remote_addr: 127.0.0.1:55071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.513 TRCE incoming request, uri: /extent/112/files, method: GET, req_id: bea0ec14-1bd7-4c55-97c3-1fc28b67600d, remote_addr: 127.0.0.1:55071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.513 INFO request completed, latency_us: 207, response_code: 200, uri: /extent/112/files, method: GET, req_id: bea0ec14-1bd7-4c55-97c3-1fc28b67600d, remote_addr: 127.0.0.1:55071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.513 INFO eid:112 Found repair files: ["070", "070.db"]
Sep 22 23:22:27.513 TRCE incoming request, uri: /newextent/112/data, method: GET, req_id: 8959e550-1dd6-4477-a20c-d59d90a0ca15, remote_addr: 127.0.0.1:55071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.514 INFO request completed, latency_us: 311, response_code: 200, uri: /newextent/112/data, method: GET, req_id: 8959e550-1dd6-4477-a20c-d59d90a0ca15, remote_addr: 127.0.0.1:55071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.519 TRCE incoming request, uri: /newextent/112/db, method: GET, req_id: fceb4789-baef-44ce-bc33-30f7b568d86c, remote_addr: 127.0.0.1:55071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.519 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/112/db, method: GET, req_id: fceb4789-baef-44ce-bc33-30f7b568d86c, remote_addr: 127.0.0.1:55071, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.520 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/070.copy" to "/tmp/downstairs-zrMnlo6G/00/000/070.replace"
Sep 22 23:22:27.520 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.521 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/070.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.521 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/070"
Sep 22 23:22:27.521 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/070.db"
Sep 22 23:22:27.521 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.521 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/070.replace" to "/tmp/downstairs-zrMnlo6G/00/000/070.completed"
Sep 22 23:22:27.521 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.521 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.522 DEBG [0] It's time to notify for 682
Sep 22 23:22:27.522 INFO Completion from [0] id:682 status:true
Sep 22 23:22:27.522 INFO [683/752] Repair commands completed
Sep 22 23:22:27.522 INFO Pop front: ReconcileIO { id: ReconciliationId(683), op: ExtentReopen { repair_id: ReconciliationId(683), extent_id: 112 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.522 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.522 INFO [0] received reconcile message
Sep 22 23:22:27.522 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(683), op: ExtentReopen { repair_id: ReconciliationId(683), extent_id: 112 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.522 INFO [0] client ExtentReopen { repair_id: ReconciliationId(683), extent_id: 112 }
Sep 22 23:22:27.522 INFO [1] received reconcile message
Sep 22 23:22:27.522 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(683), op: ExtentReopen { repair_id: ReconciliationId(683), extent_id: 112 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.522 INFO [1] client ExtentReopen { repair_id: ReconciliationId(683), extent_id: 112 }
Sep 22 23:22:27.522 INFO [2] received reconcile message
Sep 22 23:22:27.522 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(683), op: ExtentReopen { repair_id: ReconciliationId(683), extent_id: 112 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.522 INFO [2] client ExtentReopen { repair_id: ReconciliationId(683), extent_id: 112 }
Sep 22 23:22:27.522 DEBG 683 Reopen extent 112
Sep 22 23:22:27.523 DEBG 683 Reopen extent 112
Sep 22 23:22:27.523 DEBG 683 Reopen extent 112
Sep 22 23:22:27.524 DEBG [2] It's time to notify for 683
Sep 22 23:22:27.524 INFO Completion from [2] id:683 status:true
Sep 22 23:22:27.524 INFO [684/752] Repair commands completed
Sep 22 23:22:27.524 INFO Pop front: ReconcileIO { id: ReconciliationId(684), op: ExtentFlush { repair_id: ReconciliationId(684), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.524 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.524 INFO [0] received reconcile message
Sep 22 23:22:27.524 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(684), op: ExtentFlush { repair_id: ReconciliationId(684), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.524 INFO [0] client ExtentFlush { repair_id: ReconciliationId(684), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.524 INFO [1] received reconcile message
Sep 22 23:22:27.524 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(684), op: ExtentFlush { repair_id: ReconciliationId(684), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.524 INFO [1] client ExtentFlush { repair_id: ReconciliationId(684), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.524 INFO [2] received reconcile message
Sep 22 23:22:27.524 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(684), op: ExtentFlush { repair_id: ReconciliationId(684), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.524 INFO [2] client ExtentFlush { repair_id: ReconciliationId(684), extent_id: 143, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.524 DEBG 684 Flush extent 143 with f:2 g:2
Sep 22 23:22:27.524 DEBG Flush just extent 143 with f:2 and g:2
Sep 22 23:22:27.525 DEBG [1] It's time to notify for 684
Sep 22 23:22:27.525 INFO Completion from [1] id:684 status:true
Sep 22 23:22:27.525 INFO [685/752] Repair commands completed
Sep 22 23:22:27.525 INFO Pop front: ReconcileIO { id: ReconciliationId(685), op: ExtentClose { repair_id: ReconciliationId(685), extent_id: 143 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.525 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.525 INFO [0] received reconcile message
Sep 22 23:22:27.525 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(685), op: ExtentClose { repair_id: ReconciliationId(685), extent_id: 143 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.525 INFO [0] client ExtentClose { repair_id: ReconciliationId(685), extent_id: 143 }
Sep 22 23:22:27.525 INFO [1] received reconcile message
Sep 22 23:22:27.525 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(685), op: ExtentClose { repair_id: ReconciliationId(685), extent_id: 143 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.525 INFO [1] client ExtentClose { repair_id: ReconciliationId(685), extent_id: 143 }
Sep 22 23:22:27.525 INFO [2] received reconcile message
Sep 22 23:22:27.525 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(685), op: ExtentClose { repair_id: ReconciliationId(685), extent_id: 143 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.525 INFO [2] client ExtentClose { repair_id: ReconciliationId(685), extent_id: 143 }
Sep 22 23:22:27.525 DEBG 685 Close extent 143
Sep 22 23:22:27.525 DEBG 685 Close extent 143
Sep 22 23:22:27.526 DEBG 685 Close extent 143
Sep 22 23:22:27.526 DEBG [2] It's time to notify for 685
Sep 22 23:22:27.526 INFO Completion from [2] id:685 status:true
Sep 22 23:22:27.526 INFO [686/752] Repair commands completed
Sep 22 23:22:27.526 INFO Pop front: ReconcileIO { id: ReconciliationId(686), op: ExtentRepair { repair_id: ReconciliationId(686), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.526 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.526 INFO [0] received reconcile message
Sep 22 23:22:27.526 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(686), op: ExtentRepair { repair_id: ReconciliationId(686), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.526 INFO [0] client ExtentRepair { repair_id: ReconciliationId(686), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.526 INFO [0] Sending repair request ReconciliationId(686)
Sep 22 23:22:27.526 INFO [1] received reconcile message
Sep 22 23:22:27.526 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(686), op: ExtentRepair { repair_id: ReconciliationId(686), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.526 INFO [1] client ExtentRepair { repair_id: ReconciliationId(686), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.526 INFO [1] No action required ReconciliationId(686)
Sep 22 23:22:27.526 INFO [2] received reconcile message
Sep 22 23:22:27.526 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(686), op: ExtentRepair { repair_id: ReconciliationId(686), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.526 INFO [2] client ExtentRepair { repair_id: ReconciliationId(686), extent_id: 143, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.526 INFO [2] No action required ReconciliationId(686)
Sep 22 23:22:27.526 DEBG 686 Repair extent 143 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.527 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/08F.copy"
Sep 22 23:22:27.589 INFO accepted connection, remote_addr: 127.0.0.1:33091, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.589 TRCE incoming request, uri: /extent/143/files, method: GET, req_id: ac8d7628-8f70-414a-a2a9-cc729a1ede28, remote_addr: 127.0.0.1:33091, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.589 INFO request completed, latency_us: 237, response_code: 200, uri: /extent/143/files, method: GET, req_id: ac8d7628-8f70-414a-a2a9-cc729a1ede28, remote_addr: 127.0.0.1:33091, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.589 INFO eid:143 Found repair files: ["08F", "08F.db"]
Sep 22 23:22:27.590 TRCE incoming request, uri: /newextent/143/data, method: GET, req_id: 8642b366-f0ad-4753-a50f-f2062d5f09fe, remote_addr: 127.0.0.1:33091, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.590 INFO request completed, latency_us: 269, response_code: 200, uri: /newextent/143/data, method: GET, req_id: 8642b366-f0ad-4753-a50f-f2062d5f09fe, remote_addr: 127.0.0.1:33091, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.595 TRCE incoming request, uri: /newextent/143/db, method: GET, req_id: 46b91a23-6f00-4a13-9872-56a46bd3113b, remote_addr: 127.0.0.1:33091, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.595 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/143/db, method: GET, req_id: 46b91a23-6f00-4a13-9872-56a46bd3113b, remote_addr: 127.0.0.1:33091, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.596 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/08F.copy" to "/tmp/downstairs-zrMnlo6G/00/000/08F.replace"
Sep 22 23:22:27.596 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.597 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/08F.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.597 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08F"
Sep 22 23:22:27.598 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/08F.db"
Sep 22 23:22:27.598 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.598 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/08F.replace" to "/tmp/downstairs-zrMnlo6G/00/000/08F.completed"
Sep 22 23:22:27.598 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.598 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.598 DEBG [0] It's time to notify for 686
Sep 22 23:22:27.598 INFO Completion from [0] id:686 status:true
Sep 22 23:22:27.598 INFO [687/752] Repair commands completed
Sep 22 23:22:27.598 INFO Pop front: ReconcileIO { id: ReconciliationId(687), op: ExtentReopen { repair_id: ReconciliationId(687), extent_id: 143 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.598 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.598 INFO [0] received reconcile message
Sep 22 23:22:27.598 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(687), op: ExtentReopen { repair_id: ReconciliationId(687), extent_id: 143 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.598 INFO [0] client ExtentReopen { repair_id: ReconciliationId(687), extent_id: 143 }
Sep 22 23:22:27.598 INFO [1] received reconcile message
Sep 22 23:22:27.598 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(687), op: ExtentReopen { repair_id: ReconciliationId(687), extent_id: 143 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.598 INFO [1] client ExtentReopen { repair_id: ReconciliationId(687), extent_id: 143 }
Sep 22 23:22:27.598 INFO [2] received reconcile message
Sep 22 23:22:27.598 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(687), op: ExtentReopen { repair_id: ReconciliationId(687), extent_id: 143 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.598 INFO [2] client ExtentReopen { repair_id: ReconciliationId(687), extent_id: 143 }
Sep 22 23:22:27.599 DEBG 687 Reopen extent 143
Sep 22 23:22:27.599 DEBG 687 Reopen extent 143
Sep 22 23:22:27.600 DEBG 687 Reopen extent 143
Sep 22 23:22:27.600 DEBG [2] It's time to notify for 687
Sep 22 23:22:27.600 INFO Completion from [2] id:687 status:true
Sep 22 23:22:27.600 INFO [688/752] Repair commands completed
Sep 22 23:22:27.600 INFO Pop front: ReconcileIO { id: ReconciliationId(688), op: ExtentFlush { repair_id: ReconciliationId(688), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.600 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.601 INFO [0] received reconcile message
Sep 22 23:22:27.601 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(688), op: ExtentFlush { repair_id: ReconciliationId(688), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.601 INFO [0] client ExtentFlush { repair_id: ReconciliationId(688), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.601 INFO [1] received reconcile message
Sep 22 23:22:27.601 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(688), op: ExtentFlush { repair_id: ReconciliationId(688), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.601 INFO [1] client ExtentFlush { repair_id: ReconciliationId(688), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.601 INFO [2] received reconcile message
Sep 22 23:22:27.601 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(688), op: ExtentFlush { repair_id: ReconciliationId(688), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.601 INFO [2] client ExtentFlush { repair_id: ReconciliationId(688), extent_id: 151, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.601 DEBG 688 Flush extent 151 with f:2 g:2
Sep 22 23:22:27.601 DEBG Flush just extent 151 with f:2 and g:2
Sep 22 23:22:27.601 DEBG [1] It's time to notify for 688
Sep 22 23:22:27.601 INFO Completion from [1] id:688 status:true
Sep 22 23:22:27.601 INFO [689/752] Repair commands completed
Sep 22 23:22:27.601 INFO Pop front: ReconcileIO { id: ReconciliationId(689), op: ExtentClose { repair_id: ReconciliationId(689), extent_id: 151 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.601 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.601 INFO [0] received reconcile message
Sep 22 23:22:27.601 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(689), op: ExtentClose { repair_id: ReconciliationId(689), extent_id: 151 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.601 INFO [0] client ExtentClose { repair_id: ReconciliationId(689), extent_id: 151 }
Sep 22 23:22:27.601 INFO [1] received reconcile message
Sep 22 23:22:27.601 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(689), op: ExtentClose { repair_id: ReconciliationId(689), extent_id: 151 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.601 INFO [1] client ExtentClose { repair_id: ReconciliationId(689), extent_id: 151 }
Sep 22 23:22:27.601 INFO [2] received reconcile message
Sep 22 23:22:27.601 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(689), op: ExtentClose { repair_id: ReconciliationId(689), extent_id: 151 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.601 INFO [2] client ExtentClose { repair_id: ReconciliationId(689), extent_id: 151 }
Sep 22 23:22:27.601 DEBG 689 Close extent 151
Sep 22 23:22:27.602 DEBG 689 Close extent 151
Sep 22 23:22:27.602 DEBG 689 Close extent 151
Sep 22 23:22:27.602 DEBG [2] It's time to notify for 689
Sep 22 23:22:27.602 INFO Completion from [2] id:689 status:true
Sep 22 23:22:27.602 INFO [690/752] Repair commands completed
Sep 22 23:22:27.603 INFO Pop front: ReconcileIO { id: ReconciliationId(690), op: ExtentRepair { repair_id: ReconciliationId(690), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.603 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.603 INFO [0] received reconcile message
Sep 22 23:22:27.603 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(690), op: ExtentRepair { repair_id: ReconciliationId(690), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.603 INFO [0] client ExtentRepair { repair_id: ReconciliationId(690), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.603 INFO [0] Sending repair request ReconciliationId(690)
Sep 22 23:22:27.603 INFO [1] received reconcile message
Sep 22 23:22:27.603 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(690), op: ExtentRepair { repair_id: ReconciliationId(690), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.603 INFO [1] client ExtentRepair { repair_id: ReconciliationId(690), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.603 INFO [1] No action required ReconciliationId(690)
Sep 22 23:22:27.603 INFO [2] received reconcile message
Sep 22 23:22:27.603 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(690), op: ExtentRepair { repair_id: ReconciliationId(690), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.603 INFO [2] client ExtentRepair { repair_id: ReconciliationId(690), extent_id: 151, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.603 INFO [2] No action required ReconciliationId(690)
Sep 22 23:22:27.603 DEBG 690 Repair extent 151 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.603 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/097.copy"
Sep 22 23:22:27.668 INFO accepted connection, remote_addr: 127.0.0.1:63298, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.668 TRCE incoming request, uri: /extent/151/files, method: GET, req_id: 66ce9c53-4f78-44a7-9f7a-f7f1d96ab2be, remote_addr: 127.0.0.1:63298, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.668 INFO request completed, latency_us: 237, response_code: 200, uri: /extent/151/files, method: GET, req_id: 66ce9c53-4f78-44a7-9f7a-f7f1d96ab2be, remote_addr: 127.0.0.1:63298, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.669 INFO eid:151 Found repair files: ["097", "097.db"]
Sep 22 23:22:27.669 TRCE incoming request, uri: /newextent/151/data, method: GET, req_id: f3e71f3c-240d-4aef-8b48-1ad5aa208f0b, remote_addr: 127.0.0.1:63298, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.669 INFO request completed, latency_us: 332, response_code: 200, uri: /newextent/151/data, method: GET, req_id: f3e71f3c-240d-4aef-8b48-1ad5aa208f0b, remote_addr: 127.0.0.1:63298, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.674 TRCE incoming request, uri: /newextent/151/db, method: GET, req_id: 75655525-ad71-4198-8dac-cdd17c73537a, remote_addr: 127.0.0.1:63298, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.675 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/151/db, method: GET, req_id: 75655525-ad71-4198-8dac-cdd17c73537a, remote_addr: 127.0.0.1:63298, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.676 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/097.copy" to "/tmp/downstairs-zrMnlo6G/00/000/097.replace"
Sep 22 23:22:27.676 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.677 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/097.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.677 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/097"
Sep 22 23:22:27.677 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/097.db"
Sep 22 23:22:27.677 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.677 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/097.replace" to "/tmp/downstairs-zrMnlo6G/00/000/097.completed"
Sep 22 23:22:27.677 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.677 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.677 DEBG [0] It's time to notify for 690
Sep 22 23:22:27.678 INFO Completion from [0] id:690 status:true
Sep 22 23:22:27.678 INFO [691/752] Repair commands completed
Sep 22 23:22:27.678 INFO Pop front: ReconcileIO { id: ReconciliationId(691), op: ExtentReopen { repair_id: ReconciliationId(691), extent_id: 151 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.678 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.678 INFO [0] received reconcile message
Sep 22 23:22:27.678 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(691), op: ExtentReopen { repair_id: ReconciliationId(691), extent_id: 151 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.678 INFO [0] client ExtentReopen { repair_id: ReconciliationId(691), extent_id: 151 }
Sep 22 23:22:27.678 INFO [1] received reconcile message
Sep 22 23:22:27.678 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(691), op: ExtentReopen { repair_id: ReconciliationId(691), extent_id: 151 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.678 INFO [1] client ExtentReopen { repair_id: ReconciliationId(691), extent_id: 151 }
Sep 22 23:22:27.678 INFO [2] received reconcile message
Sep 22 23:22:27.678 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(691), op: ExtentReopen { repair_id: ReconciliationId(691), extent_id: 151 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.678 INFO [2] client ExtentReopen { repair_id: ReconciliationId(691), extent_id: 151 }
Sep 22 23:22:27.678 DEBG 691 Reopen extent 151
Sep 22 23:22:27.679 DEBG 691 Reopen extent 151
Sep 22 23:22:27.679 DEBG 691 Reopen extent 151
Sep 22 23:22:27.680 DEBG [2] It's time to notify for 691
Sep 22 23:22:27.680 INFO Completion from [2] id:691 status:true
Sep 22 23:22:27.680 INFO [692/752] Repair commands completed
Sep 22 23:22:27.680 INFO Pop front: ReconcileIO { id: ReconciliationId(692), op: ExtentFlush { repair_id: ReconciliationId(692), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.680 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.680 INFO [0] received reconcile message
Sep 22 23:22:27.680 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(692), op: ExtentFlush { repair_id: ReconciliationId(692), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.680 INFO [0] client ExtentFlush { repair_id: ReconciliationId(692), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.680 INFO [1] received reconcile message
Sep 22 23:22:27.680 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(692), op: ExtentFlush { repair_id: ReconciliationId(692), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.680 INFO [1] client ExtentFlush { repair_id: ReconciliationId(692), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.680 INFO [2] received reconcile message
Sep 22 23:22:27.680 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(692), op: ExtentFlush { repair_id: ReconciliationId(692), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.680 INFO [2] client ExtentFlush { repair_id: ReconciliationId(692), extent_id: 101, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.680 DEBG 692 Flush extent 101 with f:2 g:2
Sep 22 23:22:27.680 DEBG Flush just extent 101 with f:2 and g:2
Sep 22 23:22:27.680 DEBG [1] It's time to notify for 692
Sep 22 23:22:27.681 INFO Completion from [1] id:692 status:true
Sep 22 23:22:27.681 INFO [693/752] Repair commands completed
Sep 22 23:22:27.681 INFO Pop front: ReconcileIO { id: ReconciliationId(693), op: ExtentClose { repair_id: ReconciliationId(693), extent_id: 101 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.681 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.681 INFO [0] received reconcile message
Sep 22 23:22:27.681 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(693), op: ExtentClose { repair_id: ReconciliationId(693), extent_id: 101 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.681 INFO [0] client ExtentClose { repair_id: ReconciliationId(693), extent_id: 101 }
Sep 22 23:22:27.681 INFO [1] received reconcile message
Sep 22 23:22:27.681 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(693), op: ExtentClose { repair_id: ReconciliationId(693), extent_id: 101 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.681 INFO [1] client ExtentClose { repair_id: ReconciliationId(693), extent_id: 101 }
Sep 22 23:22:27.681 INFO [2] received reconcile message
Sep 22 23:22:27.681 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(693), op: ExtentClose { repair_id: ReconciliationId(693), extent_id: 101 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.681 INFO [2] client ExtentClose { repair_id: ReconciliationId(693), extent_id: 101 }
Sep 22 23:22:27.681 DEBG 693 Close extent 101
Sep 22 23:22:27.681 DEBG 693 Close extent 101
Sep 22 23:22:27.682 DEBG 693 Close extent 101
Sep 22 23:22:27.682 DEBG [2] It's time to notify for 693
Sep 22 23:22:27.682 INFO Completion from [2] id:693 status:true
Sep 22 23:22:27.682 INFO [694/752] Repair commands completed
Sep 22 23:22:27.682 INFO Pop front: ReconcileIO { id: ReconciliationId(694), op: ExtentRepair { repair_id: ReconciliationId(694), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.682 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.682 INFO [0] received reconcile message
Sep 22 23:22:27.682 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(694), op: ExtentRepair { repair_id: ReconciliationId(694), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.682 INFO [0] client ExtentRepair { repair_id: ReconciliationId(694), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.682 INFO [0] Sending repair request ReconciliationId(694)
Sep 22 23:22:27.682 INFO [1] received reconcile message
Sep 22 23:22:27.682 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(694), op: ExtentRepair { repair_id: ReconciliationId(694), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.682 INFO [1] client ExtentRepair { repair_id: ReconciliationId(694), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.682 INFO [1] No action required ReconciliationId(694)
Sep 22 23:22:27.682 INFO [2] received reconcile message
Sep 22 23:22:27.682 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(694), op: ExtentRepair { repair_id: ReconciliationId(694), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.682 INFO [2] client ExtentRepair { repair_id: ReconciliationId(694), extent_id: 101, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.682 INFO [2] No action required ReconciliationId(694)
Sep 22 23:22:27.682 DEBG 694 Repair extent 101 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.683 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/065.copy"
Sep 22 23:22:27.747 INFO accepted connection, remote_addr: 127.0.0.1:37015, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.747 TRCE incoming request, uri: /extent/101/files, method: GET, req_id: 1b764a04-d2b2-4cdd-98f0-f5e24d54eb18, remote_addr: 127.0.0.1:37015, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.748 INFO request completed, latency_us: 230, response_code: 200, uri: /extent/101/files, method: GET, req_id: 1b764a04-d2b2-4cdd-98f0-f5e24d54eb18, remote_addr: 127.0.0.1:37015, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.748 INFO eid:101 Found repair files: ["065", "065.db"]
Sep 22 23:22:27.748 TRCE incoming request, uri: /newextent/101/data, method: GET, req_id: 2705c68f-7de3-40d3-b657-db8f42ee9cb6, remote_addr: 127.0.0.1:37015, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.749 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/101/data, method: GET, req_id: 2705c68f-7de3-40d3-b657-db8f42ee9cb6, remote_addr: 127.0.0.1:37015, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.753 TRCE incoming request, uri: /newextent/101/db, method: GET, req_id: 7240f1e3-98f3-4626-95d6-68d21b65d73c, remote_addr: 127.0.0.1:37015, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.754 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/101/db, method: GET, req_id: 7240f1e3-98f3-4626-95d6-68d21b65d73c, remote_addr: 127.0.0.1:37015, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.755 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/065.copy" to "/tmp/downstairs-zrMnlo6G/00/000/065.replace"
Sep 22 23:22:27.755 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.756 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/065.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.756 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/065"
Sep 22 23:22:27.756 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/065.db"
Sep 22 23:22:27.756 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.756 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/065.replace" to "/tmp/downstairs-zrMnlo6G/00/000/065.completed"
Sep 22 23:22:27.756 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.756 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.756 DEBG [0] It's time to notify for 694
Sep 22 23:22:27.757 INFO Completion from [0] id:694 status:true
Sep 22 23:22:27.757 INFO [695/752] Repair commands completed
Sep 22 23:22:27.757 INFO Pop front: ReconcileIO { id: ReconciliationId(695), op: ExtentReopen { repair_id: ReconciliationId(695), extent_id: 101 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.757 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.757 INFO [0] received reconcile message
Sep 22 23:22:27.757 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(695), op: ExtentReopen { repair_id: ReconciliationId(695), extent_id: 101 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.757 INFO [0] client ExtentReopen { repair_id: ReconciliationId(695), extent_id: 101 }
Sep 22 23:22:27.757 INFO [1] received reconcile message
Sep 22 23:22:27.757 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(695), op: ExtentReopen { repair_id: ReconciliationId(695), extent_id: 101 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.757 INFO [1] client ExtentReopen { repair_id: ReconciliationId(695), extent_id: 101 }
Sep 22 23:22:27.757 INFO [2] received reconcile message
Sep 22 23:22:27.757 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(695), op: ExtentReopen { repair_id: ReconciliationId(695), extent_id: 101 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.757 INFO [2] client ExtentReopen { repair_id: ReconciliationId(695), extent_id: 101 }
Sep 22 23:22:27.757 DEBG 695 Reopen extent 101
Sep 22 23:22:27.758 DEBG 695 Reopen extent 101
Sep 22 23:22:27.758 DEBG 695 Reopen extent 101
Sep 22 23:22:27.759 DEBG [2] It's time to notify for 695
Sep 22 23:22:27.759 INFO Completion from [2] id:695 status:true
Sep 22 23:22:27.759 INFO [696/752] Repair commands completed
Sep 22 23:22:27.759 INFO Pop front: ReconcileIO { id: ReconciliationId(696), op: ExtentFlush { repair_id: ReconciliationId(696), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.759 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.759 INFO [0] received reconcile message
Sep 22 23:22:27.759 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(696), op: ExtentFlush { repair_id: ReconciliationId(696), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.759 INFO [0] client ExtentFlush { repair_id: ReconciliationId(696), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.759 INFO [1] received reconcile message
Sep 22 23:22:27.759 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(696), op: ExtentFlush { repair_id: ReconciliationId(696), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.759 INFO [1] client ExtentFlush { repair_id: ReconciliationId(696), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.759 INFO [2] received reconcile message
Sep 22 23:22:27.759 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(696), op: ExtentFlush { repair_id: ReconciliationId(696), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.759 INFO [2] client ExtentFlush { repair_id: ReconciliationId(696), extent_id: 173, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.759 DEBG 696 Flush extent 173 with f:2 g:2
Sep 22 23:22:27.759 DEBG Flush just extent 173 with f:2 and g:2
Sep 22 23:22:27.759 DEBG [1] It's time to notify for 696
Sep 22 23:22:27.759 INFO Completion from [1] id:696 status:true
Sep 22 23:22:27.760 INFO [697/752] Repair commands completed
Sep 22 23:22:27.760 INFO Pop front: ReconcileIO { id: ReconciliationId(697), op: ExtentClose { repair_id: ReconciliationId(697), extent_id: 173 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.760 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.760 INFO [0] received reconcile message
Sep 22 23:22:27.760 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(697), op: ExtentClose { repair_id: ReconciliationId(697), extent_id: 173 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.760 INFO [0] client ExtentClose { repair_id: ReconciliationId(697), extent_id: 173 }
Sep 22 23:22:27.760 INFO [1] received reconcile message
Sep 22 23:22:27.760 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(697), op: ExtentClose { repair_id: ReconciliationId(697), extent_id: 173 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.760 INFO [1] client ExtentClose { repair_id: ReconciliationId(697), extent_id: 173 }
Sep 22 23:22:27.760 INFO [2] received reconcile message
Sep 22 23:22:27.760 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(697), op: ExtentClose { repair_id: ReconciliationId(697), extent_id: 173 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.760 INFO [2] client ExtentClose { repair_id: ReconciliationId(697), extent_id: 173 }
Sep 22 23:22:27.760 DEBG 697 Close extent 173
Sep 22 23:22:27.760 DEBG 697 Close extent 173
Sep 22 23:22:27.760 DEBG 697 Close extent 173
Sep 22 23:22:27.761 DEBG [2] It's time to notify for 697
Sep 22 23:22:27.761 INFO Completion from [2] id:697 status:true
Sep 22 23:22:27.761 INFO [698/752] Repair commands completed
Sep 22 23:22:27.761 INFO Pop front: ReconcileIO { id: ReconciliationId(698), op: ExtentRepair { repair_id: ReconciliationId(698), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.761 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.761 INFO [0] received reconcile message
Sep 22 23:22:27.761 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(698), op: ExtentRepair { repair_id: ReconciliationId(698), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.761 INFO [0] client ExtentRepair { repair_id: ReconciliationId(698), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.761 INFO [0] Sending repair request ReconciliationId(698)
Sep 22 23:22:27.761 INFO [1] received reconcile message
Sep 22 23:22:27.761 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(698), op: ExtentRepair { repair_id: ReconciliationId(698), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.761 INFO [1] client ExtentRepair { repair_id: ReconciliationId(698), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.761 INFO [1] No action required ReconciliationId(698)
Sep 22 23:22:27.761 INFO [2] received reconcile message
Sep 22 23:22:27.761 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(698), op: ExtentRepair { repair_id: ReconciliationId(698), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.761 INFO [2] client ExtentRepair { repair_id: ReconciliationId(698), extent_id: 173, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.761 INFO [2] No action required ReconciliationId(698)
Sep 22 23:22:27.761 DEBG 698 Repair extent 173 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.761 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0AD.copy"
Sep 22 23:22:27.817 DEBG [0] Read AckReady 1093, : downstairs
Sep 22 23:22:27.827 INFO accepted connection, remote_addr: 127.0.0.1:64338, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.827 TRCE incoming request, uri: /extent/173/files, method: GET, req_id: 841e9673-d182-483b-abf8-71e0731fd45a, remote_addr: 127.0.0.1:64338, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.827 INFO request completed, latency_us: 280, response_code: 200, uri: /extent/173/files, method: GET, req_id: 841e9673-d182-483b-abf8-71e0731fd45a, remote_addr: 127.0.0.1:64338, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.827 INFO eid:173 Found repair files: ["0AD", "0AD.db"]
Sep 22 23:22:27.828 TRCE incoming request, uri: /newextent/173/data, method: GET, req_id: 3a4a6b3c-7faf-4a2c-80d4-10450327c20f, remote_addr: 127.0.0.1:64338, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.828 INFO request completed, latency_us: 382, response_code: 200, uri: /newextent/173/data, method: GET, req_id: 3a4a6b3c-7faf-4a2c-80d4-10450327c20f, remote_addr: 127.0.0.1:64338, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.833 TRCE incoming request, uri: /newextent/173/db, method: GET, req_id: 0fc68043-49c8-4d25-90ac-61bbc17952e4, remote_addr: 127.0.0.1:64338, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.833 INFO request completed, latency_us: 309, response_code: 200, uri: /newextent/173/db, method: GET, req_id: 0fc68043-49c8-4d25-90ac-61bbc17952e4, remote_addr: 127.0.0.1:64338, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.834 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0AD.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0AD.replace"
Sep 22 23:22:27.834 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.836 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0AD.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.836 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AD"
Sep 22 23:22:27.836 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AD.db"
Sep 22 23:22:27.836 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.836 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0AD.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0AD.completed"
Sep 22 23:22:27.837 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.837 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.837 DEBG [0] It's time to notify for 698
Sep 22 23:22:27.837 INFO Completion from [0] id:698 status:true
Sep 22 23:22:27.837 INFO [699/752] Repair commands completed
Sep 22 23:22:27.837 INFO Pop front: ReconcileIO { id: ReconciliationId(699), op: ExtentReopen { repair_id: ReconciliationId(699), extent_id: 173 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.837 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.837 INFO [0] received reconcile message
Sep 22 23:22:27.837 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(699), op: ExtentReopen { repair_id: ReconciliationId(699), extent_id: 173 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.837 INFO [0] client ExtentReopen { repair_id: ReconciliationId(699), extent_id: 173 }
Sep 22 23:22:27.837 INFO [1] received reconcile message
Sep 22 23:22:27.837 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(699), op: ExtentReopen { repair_id: ReconciliationId(699), extent_id: 173 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.837 INFO [1] client ExtentReopen { repair_id: ReconciliationId(699), extent_id: 173 }
Sep 22 23:22:27.837 INFO [2] received reconcile message
Sep 22 23:22:27.837 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(699), op: ExtentReopen { repair_id: ReconciliationId(699), extent_id: 173 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.837 INFO [2] client ExtentReopen { repair_id: ReconciliationId(699), extent_id: 173 }
Sep 22 23:22:27.838 DEBG 699 Reopen extent 173
Sep 22 23:22:27.838 DEBG 699 Reopen extent 173
Sep 22 23:22:27.839 DEBG 699 Reopen extent 173
Sep 22 23:22:27.839 DEBG [2] It's time to notify for 699
Sep 22 23:22:27.840 INFO Completion from [2] id:699 status:true
Sep 22 23:22:27.840 INFO [700/752] Repair commands completed
Sep 22 23:22:27.840 INFO Pop front: ReconcileIO { id: ReconciliationId(700), op: ExtentFlush { repair_id: ReconciliationId(700), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.840 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.840 INFO [0] received reconcile message
Sep 22 23:22:27.840 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(700), op: ExtentFlush { repair_id: ReconciliationId(700), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.840 INFO [0] client ExtentFlush { repair_id: ReconciliationId(700), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.840 INFO [1] received reconcile message
Sep 22 23:22:27.840 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(700), op: ExtentFlush { repair_id: ReconciliationId(700), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.840 INFO [1] client ExtentFlush { repair_id: ReconciliationId(700), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.840 INFO [2] received reconcile message
Sep 22 23:22:27.840 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(700), op: ExtentFlush { repair_id: ReconciliationId(700), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.840 INFO [2] client ExtentFlush { repair_id: ReconciliationId(700), extent_id: 20, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.840 DEBG 700 Flush extent 20 with f:2 g:2
Sep 22 23:22:27.840 DEBG Flush just extent 20 with f:2 and g:2
Sep 22 23:22:27.840 DEBG [1] It's time to notify for 700
Sep 22 23:22:27.840 INFO Completion from [1] id:700 status:true
Sep 22 23:22:27.840 INFO [701/752] Repair commands completed
Sep 22 23:22:27.840 INFO Pop front: ReconcileIO { id: ReconciliationId(701), op: ExtentClose { repair_id: ReconciliationId(701), extent_id: 20 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.840 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.840 INFO [0] received reconcile message
Sep 22 23:22:27.840 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(701), op: ExtentClose { repair_id: ReconciliationId(701), extent_id: 20 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.840 INFO [0] client ExtentClose { repair_id: ReconciliationId(701), extent_id: 20 }
Sep 22 23:22:27.840 INFO [1] received reconcile message
Sep 22 23:22:27.840 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(701), op: ExtentClose { repair_id: ReconciliationId(701), extent_id: 20 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.840 INFO [1] client ExtentClose { repair_id: ReconciliationId(701), extent_id: 20 }
Sep 22 23:22:27.840 INFO [2] received reconcile message
Sep 22 23:22:27.840 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(701), op: ExtentClose { repair_id: ReconciliationId(701), extent_id: 20 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.840 INFO [2] client ExtentClose { repair_id: ReconciliationId(701), extent_id: 20 }
Sep 22 23:22:27.841 DEBG 701 Close extent 20
Sep 22 23:22:27.841 DEBG 701 Close extent 20
Sep 22 23:22:27.841 DEBG 701 Close extent 20
Sep 22 23:22:27.842 DEBG [2] It's time to notify for 701
Sep 22 23:22:27.842 INFO Completion from [2] id:701 status:true
Sep 22 23:22:27.842 INFO [702/752] Repair commands completed
Sep 22 23:22:27.842 INFO Pop front: ReconcileIO { id: ReconciliationId(702), op: ExtentRepair { repair_id: ReconciliationId(702), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.842 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.842 INFO [0] received reconcile message
Sep 22 23:22:27.842 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(702), op: ExtentRepair { repair_id: ReconciliationId(702), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.842 INFO [0] client ExtentRepair { repair_id: ReconciliationId(702), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.842 INFO [0] Sending repair request ReconciliationId(702)
Sep 22 23:22:27.842 INFO [1] received reconcile message
Sep 22 23:22:27.842 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(702), op: ExtentRepair { repair_id: ReconciliationId(702), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.842 INFO [1] client ExtentRepair { repair_id: ReconciliationId(702), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.842 INFO [1] No action required ReconciliationId(702)
Sep 22 23:22:27.842 INFO [2] received reconcile message
Sep 22 23:22:27.842 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(702), op: ExtentRepair { repair_id: ReconciliationId(702), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.842 INFO [2] client ExtentRepair { repair_id: ReconciliationId(702), extent_id: 20, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.842 INFO [2] No action required ReconciliationId(702)
Sep 22 23:22:27.842 DEBG 702 Repair extent 20 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.842 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/014.copy"
Sep 22 23:22:27.909 INFO accepted connection, remote_addr: 127.0.0.1:37947, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.909 TRCE incoming request, uri: /extent/20/files, method: GET, req_id: 6038f6e2-787e-46e6-86c4-596823d9314e, remote_addr: 127.0.0.1:37947, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.910 INFO request completed, latency_us: 283, response_code: 200, uri: /extent/20/files, method: GET, req_id: 6038f6e2-787e-46e6-86c4-596823d9314e, remote_addr: 127.0.0.1:37947, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.910 INFO eid:20 Found repair files: ["014", "014.db"]
Sep 22 23:22:27.910 TRCE incoming request, uri: /newextent/20/data, method: GET, req_id: fada309d-928c-48db-908a-2f6b2c0a2044, remote_addr: 127.0.0.1:37947, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.911 INFO request completed, latency_us: 378, response_code: 200, uri: /newextent/20/data, method: GET, req_id: fada309d-928c-48db-908a-2f6b2c0a2044, remote_addr: 127.0.0.1:37947, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.916 TRCE incoming request, uri: /newextent/20/db, method: GET, req_id: c0205d8e-3a61-47ff-8b8c-7b12b2332b1d, remote_addr: 127.0.0.1:37947, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.916 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/20/db, method: GET, req_id: c0205d8e-3a61-47ff-8b8c-7b12b2332b1d, remote_addr: 127.0.0.1:37947, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.917 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/014.copy" to "/tmp/downstairs-zrMnlo6G/00/000/014.replace"
Sep 22 23:22:27.917 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.918 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/014.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.919 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/014"
Sep 22 23:22:27.919 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/014.db"
Sep 22 23:22:27.919 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.919 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/014.replace" to "/tmp/downstairs-zrMnlo6G/00/000/014.completed"
Sep 22 23:22:27.919 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.919 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.919 DEBG [0] It's time to notify for 702
Sep 22 23:22:27.919 INFO Completion from [0] id:702 status:true
Sep 22 23:22:27.919 INFO [703/752] Repair commands completed
Sep 22 23:22:27.919 INFO Pop front: ReconcileIO { id: ReconciliationId(703), op: ExtentReopen { repair_id: ReconciliationId(703), extent_id: 20 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.919 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.919 INFO [0] received reconcile message
Sep 22 23:22:27.919 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(703), op: ExtentReopen { repair_id: ReconciliationId(703), extent_id: 20 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.919 INFO [0] client ExtentReopen { repair_id: ReconciliationId(703), extent_id: 20 }
Sep 22 23:22:27.920 INFO [1] received reconcile message
Sep 22 23:22:27.920 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(703), op: ExtentReopen { repair_id: ReconciliationId(703), extent_id: 20 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.920 INFO [1] client ExtentReopen { repair_id: ReconciliationId(703), extent_id: 20 }
Sep 22 23:22:27.920 INFO [2] received reconcile message
Sep 22 23:22:27.920 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(703), op: ExtentReopen { repair_id: ReconciliationId(703), extent_id: 20 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.920 INFO [2] client ExtentReopen { repair_id: ReconciliationId(703), extent_id: 20 }
Sep 22 23:22:27.920 DEBG 703 Reopen extent 20
Sep 22 23:22:27.921 DEBG 703 Reopen extent 20
Sep 22 23:22:27.921 DEBG 703 Reopen extent 20
Sep 22 23:22:27.922 DEBG [2] It's time to notify for 703
Sep 22 23:22:27.922 INFO Completion from [2] id:703 status:true
Sep 22 23:22:27.922 INFO [704/752] Repair commands completed
Sep 22 23:22:27.922 INFO Pop front: ReconcileIO { id: ReconciliationId(704), op: ExtentFlush { repair_id: ReconciliationId(704), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.922 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.922 INFO [0] received reconcile message
Sep 22 23:22:27.922 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(704), op: ExtentFlush { repair_id: ReconciliationId(704), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.922 INFO [0] client ExtentFlush { repair_id: ReconciliationId(704), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.922 INFO [1] received reconcile message
Sep 22 23:22:27.922 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(704), op: ExtentFlush { repair_id: ReconciliationId(704), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:27.922 INFO [1] client ExtentFlush { repair_id: ReconciliationId(704), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.922 INFO [2] received reconcile message
Sep 22 23:22:27.922 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(704), op: ExtentFlush { repair_id: ReconciliationId(704), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.922 INFO [2] client ExtentFlush { repair_id: ReconciliationId(704), extent_id: 104, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:27.922 DEBG 704 Flush extent 104 with f:2 g:2
Sep 22 23:22:27.922 DEBG Flush just extent 104 with f:2 and g:2
Sep 22 23:22:27.922 DEBG [1] It's time to notify for 704
Sep 22 23:22:27.922 INFO Completion from [1] id:704 status:true
Sep 22 23:22:27.922 INFO [705/752] Repair commands completed
Sep 22 23:22:27.922 INFO Pop front: ReconcileIO { id: ReconciliationId(705), op: ExtentClose { repair_id: ReconciliationId(705), extent_id: 104 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.923 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.923 INFO [0] received reconcile message
Sep 22 23:22:27.923 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(705), op: ExtentClose { repair_id: ReconciliationId(705), extent_id: 104 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.923 INFO [0] client ExtentClose { repair_id: ReconciliationId(705), extent_id: 104 }
Sep 22 23:22:27.923 INFO [1] received reconcile message
Sep 22 23:22:27.923 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(705), op: ExtentClose { repair_id: ReconciliationId(705), extent_id: 104 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.923 INFO [1] client ExtentClose { repair_id: ReconciliationId(705), extent_id: 104 }
Sep 22 23:22:27.923 INFO [2] received reconcile message
Sep 22 23:22:27.923 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(705), op: ExtentClose { repair_id: ReconciliationId(705), extent_id: 104 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.923 INFO [2] client ExtentClose { repair_id: ReconciliationId(705), extent_id: 104 }
Sep 22 23:22:27.923 DEBG 705 Close extent 104
Sep 22 23:22:27.923 DEBG 705 Close extent 104
Sep 22 23:22:27.923 DEBG 705 Close extent 104
Sep 22 23:22:27.924 DEBG [2] It's time to notify for 705
Sep 22 23:22:27.924 INFO Completion from [2] id:705 status:true
Sep 22 23:22:27.924 INFO [706/752] Repair commands completed
Sep 22 23:22:27.924 INFO Pop front: ReconcileIO { id: ReconciliationId(706), op: ExtentRepair { repair_id: ReconciliationId(706), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.924 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.924 INFO [0] received reconcile message
Sep 22 23:22:27.924 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(706), op: ExtentRepair { repair_id: ReconciliationId(706), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.924 INFO [0] client ExtentRepair { repair_id: ReconciliationId(706), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.924 INFO [0] Sending repair request ReconciliationId(706)
Sep 22 23:22:27.924 INFO [1] received reconcile message
Sep 22 23:22:27.924 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(706), op: ExtentRepair { repair_id: ReconciliationId(706), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.924 INFO [1] client ExtentRepair { repair_id: ReconciliationId(706), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.924 INFO [1] No action required ReconciliationId(706)
Sep 22 23:22:27.924 INFO [2] received reconcile message
Sep 22 23:22:27.924 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(706), op: ExtentRepair { repair_id: ReconciliationId(706), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:27.924 INFO [2] client ExtentRepair { repair_id: ReconciliationId(706), extent_id: 104, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:27.924 INFO [2] No action required ReconciliationId(706)
Sep 22 23:22:27.924 DEBG 706 Repair extent 104 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:27.924 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/068.copy"
Sep 22 23:22:27.989 INFO accepted connection, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.989 TRCE incoming request, uri: /extent/104/files, method: GET, req_id: 9382c064-ece8-4d33-9ac3-1504eeead0b0, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.990 INFO request completed, latency_us: 222, response_code: 200, uri: /extent/104/files, method: GET, req_id: 9382c064-ece8-4d33-9ac3-1504eeead0b0, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.990 INFO eid:104 Found repair files: ["068", "068.db"]
Sep 22 23:22:27.990 TRCE incoming request, uri: /newextent/104/data, method: GET, req_id: cd7f2879-b231-47a1-9626-585d0c29c482, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.991 INFO request completed, latency_us: 324, response_code: 200, uri: /newextent/104/data, method: GET, req_id: cd7f2879-b231-47a1-9626-585d0c29c482, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.996 TRCE incoming request, uri: /newextent/104/db, method: GET, req_id: 6008e912-1579-4c14-b4de-a6510e50cdc0, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.996 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/104/db, method: GET, req_id: 6008e912-1579-4c14-b4de-a6510e50cdc0, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:27.997 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/068.copy" to "/tmp/downstairs-zrMnlo6G/00/000/068.replace"
Sep 22 23:22:27.997 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.998 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/068.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.998 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/068"
Sep 22 23:22:27.998 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/068.db"
Sep 22 23:22:27.998 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.998 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/068.replace" to "/tmp/downstairs-zrMnlo6G/00/000/068.completed"
Sep 22 23:22:27.998 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.998 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:27.999 DEBG [0] It's time to notify for 706
Sep 22 23:22:27.999 INFO Completion from [0] id:706 status:true
Sep 22 23:22:27.999 INFO [707/752] Repair commands completed
Sep 22 23:22:27.999 INFO Pop front: ReconcileIO { id: ReconciliationId(707), op: ExtentReopen { repair_id: ReconciliationId(707), extent_id: 104 }, state: ClientData([New, New, New]) }
Sep 22 23:22:27.999 INFO Sent repair work, now wait for resp
Sep 22 23:22:27.999 INFO [0] received reconcile message
Sep 22 23:22:27.999 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(707), op: ExtentReopen { repair_id: ReconciliationId(707), extent_id: 104 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:27.999 INFO [0] client ExtentReopen { repair_id: ReconciliationId(707), extent_id: 104 }
Sep 22 23:22:27.999 INFO [1] received reconcile message
Sep 22 23:22:27.999 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(707), op: ExtentReopen { repair_id: ReconciliationId(707), extent_id: 104 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:27.999 INFO [1] client ExtentReopen { repair_id: ReconciliationId(707), extent_id: 104 }
Sep 22 23:22:27.999 INFO [2] received reconcile message
Sep 22 23:22:27.999 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(707), op: ExtentReopen { repair_id: ReconciliationId(707), extent_id: 104 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:27.999 INFO [2] client ExtentReopen { repair_id: ReconciliationId(707), extent_id: 104 }
Sep 22 23:22:27.999 DEBG 707 Reopen extent 104
Sep 22 23:22:28.000 DEBG 707 Reopen extent 104
Sep 22 23:22:28.000 DEBG 707 Reopen extent 104
Sep 22 23:22:28.001 DEBG [2] It's time to notify for 707
Sep 22 23:22:28.001 INFO Completion from [2] id:707 status:true
Sep 22 23:22:28.001 INFO [708/752] Repair commands completed
Sep 22 23:22:28.001 INFO Pop front: ReconcileIO { id: ReconciliationId(708), op: ExtentFlush { repair_id: ReconciliationId(708), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.001 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.001 INFO [0] received reconcile message
Sep 22 23:22:28.001 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(708), op: ExtentFlush { repair_id: ReconciliationId(708), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.001 INFO [0] client ExtentFlush { repair_id: ReconciliationId(708), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.001 INFO [1] received reconcile message
Sep 22 23:22:28.001 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(708), op: ExtentFlush { repair_id: ReconciliationId(708), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.001 INFO [1] client ExtentFlush { repair_id: ReconciliationId(708), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.001 INFO [2] received reconcile message
Sep 22 23:22:28.001 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(708), op: ExtentFlush { repair_id: ReconciliationId(708), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.001 INFO [2] client ExtentFlush { repair_id: ReconciliationId(708), extent_id: 81, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.002 DEBG 708 Flush extent 81 with f:2 g:2
Sep 22 23:22:28.002 DEBG Flush just extent 81 with f:2 and g:2
Sep 22 23:22:28.002 DEBG [1] It's time to notify for 708
Sep 22 23:22:28.002 INFO Completion from [1] id:708 status:true
Sep 22 23:22:28.002 INFO [709/752] Repair commands completed
Sep 22 23:22:28.002 INFO Pop front: ReconcileIO { id: ReconciliationId(709), op: ExtentClose { repair_id: ReconciliationId(709), extent_id: 81 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.002 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.002 INFO [0] received reconcile message
Sep 22 23:22:28.002 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(709), op: ExtentClose { repair_id: ReconciliationId(709), extent_id: 81 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.002 INFO [0] client ExtentClose { repair_id: ReconciliationId(709), extent_id: 81 }
Sep 22 23:22:28.002 INFO [1] received reconcile message
Sep 22 23:22:28.002 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(709), op: ExtentClose { repair_id: ReconciliationId(709), extent_id: 81 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.002 INFO [1] client ExtentClose { repair_id: ReconciliationId(709), extent_id: 81 }
Sep 22 23:22:28.002 INFO [2] received reconcile message
Sep 22 23:22:28.002 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(709), op: ExtentClose { repair_id: ReconciliationId(709), extent_id: 81 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.002 INFO [2] client ExtentClose { repair_id: ReconciliationId(709), extent_id: 81 }
Sep 22 23:22:28.002 DEBG 709 Close extent 81
Sep 22 23:22:28.002 DEBG 709 Close extent 81
Sep 22 23:22:28.003 DEBG 709 Close extent 81
Sep 22 23:22:28.003 DEBG [2] It's time to notify for 709
Sep 22 23:22:28.003 INFO Completion from [2] id:709 status:true
Sep 22 23:22:28.003 INFO [710/752] Repair commands completed
Sep 22 23:22:28.003 INFO Pop front: ReconcileIO { id: ReconciliationId(710), op: ExtentRepair { repair_id: ReconciliationId(710), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.003 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.003 INFO [0] received reconcile message
Sep 22 23:22:28.003 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(710), op: ExtentRepair { repair_id: ReconciliationId(710), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.003 INFO [0] client ExtentRepair { repair_id: ReconciliationId(710), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.003 INFO [0] Sending repair request ReconciliationId(710)
Sep 22 23:22:28.003 INFO [1] received reconcile message
Sep 22 23:22:28.003 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(710), op: ExtentRepair { repair_id: ReconciliationId(710), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.003 INFO [1] client ExtentRepair { repair_id: ReconciliationId(710), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.003 INFO [1] No action required ReconciliationId(710)
Sep 22 23:22:28.003 INFO [2] received reconcile message
Sep 22 23:22:28.004 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(710), op: ExtentRepair { repair_id: ReconciliationId(710), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.004 INFO [2] client ExtentRepair { repair_id: ReconciliationId(710), extent_id: 81, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.004 INFO [2] No action required ReconciliationId(710)
Sep 22 23:22:28.004 DEBG 710 Repair extent 81 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.004 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/051.copy"
Sep 22 23:22:28.069 INFO accepted connection, remote_addr: 127.0.0.1:48474, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.070 TRCE incoming request, uri: /extent/81/files, method: GET, req_id: c8331010-c2da-45ca-b56b-984b4c25f239, remote_addr: 127.0.0.1:48474, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.070 INFO request completed, latency_us: 246, response_code: 200, uri: /extent/81/files, method: GET, req_id: c8331010-c2da-45ca-b56b-984b4c25f239, remote_addr: 127.0.0.1:48474, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.070 INFO eid:81 Found repair files: ["051", "051.db"]
Sep 22 23:22:28.070 TRCE incoming request, uri: /newextent/81/data, method: GET, req_id: 28992663-e63f-4cee-b965-005b9d7740f3, remote_addr: 127.0.0.1:48474, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.071 INFO request completed, latency_us: 358, response_code: 200, uri: /newextent/81/data, method: GET, req_id: 28992663-e63f-4cee-b965-005b9d7740f3, remote_addr: 127.0.0.1:48474, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.076 TRCE incoming request, uri: /newextent/81/db, method: GET, req_id: da08371f-3635-4080-b01a-4fbd9df28301, remote_addr: 127.0.0.1:48474, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.076 INFO request completed, latency_us: 303, response_code: 200, uri: /newextent/81/db, method: GET, req_id: da08371f-3635-4080-b01a-4fbd9df28301, remote_addr: 127.0.0.1:48474, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.077 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/051.copy" to "/tmp/downstairs-zrMnlo6G/00/000/051.replace"
Sep 22 23:22:28.077 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.078 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/051.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.078 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/051"
Sep 22 23:22:28.078 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/051.db"
Sep 22 23:22:28.078 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.078 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/051.replace" to "/tmp/downstairs-zrMnlo6G/00/000/051.completed"
Sep 22 23:22:28.078 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.078 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.079 DEBG [0] It's time to notify for 710
Sep 22 23:22:28.079 INFO Completion from [0] id:710 status:true
Sep 22 23:22:28.079 INFO [711/752] Repair commands completed
Sep 22 23:22:28.079 INFO Pop front: ReconcileIO { id: ReconciliationId(711), op: ExtentReopen { repair_id: ReconciliationId(711), extent_id: 81 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.079 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.079 INFO [0] received reconcile message
Sep 22 23:22:28.079 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(711), op: ExtentReopen { repair_id: ReconciliationId(711), extent_id: 81 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.079 INFO [0] client ExtentReopen { repair_id: ReconciliationId(711), extent_id: 81 }
Sep 22 23:22:28.079 INFO [1] received reconcile message
Sep 22 23:22:28.079 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(711), op: ExtentReopen { repair_id: ReconciliationId(711), extent_id: 81 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.079 INFO [1] client ExtentReopen { repair_id: ReconciliationId(711), extent_id: 81 }
Sep 22 23:22:28.079 INFO [2] received reconcile message
Sep 22 23:22:28.079 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(711), op: ExtentReopen { repair_id: ReconciliationId(711), extent_id: 81 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.079 INFO [2] client ExtentReopen { repair_id: ReconciliationId(711), extent_id: 81 }
Sep 22 23:22:28.079 DEBG 711 Reopen extent 81
Sep 22 23:22:28.080 DEBG 711 Reopen extent 81
Sep 22 23:22:28.080 DEBG 711 Reopen extent 81
Sep 22 23:22:28.081 DEBG [2] It's time to notify for 711
Sep 22 23:22:28.081 INFO Completion from [2] id:711 status:true
Sep 22 23:22:28.081 INFO [712/752] Repair commands completed
Sep 22 23:22:28.081 INFO Pop front: ReconcileIO { id: ReconciliationId(712), op: ExtentFlush { repair_id: ReconciliationId(712), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.081 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.081 INFO [0] received reconcile message
Sep 22 23:22:28.081 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(712), op: ExtentFlush { repair_id: ReconciliationId(712), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.081 INFO [0] client ExtentFlush { repair_id: ReconciliationId(712), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.081 INFO [1] received reconcile message
Sep 22 23:22:28.081 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(712), op: ExtentFlush { repair_id: ReconciliationId(712), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.081 INFO [1] client ExtentFlush { repair_id: ReconciliationId(712), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.081 INFO [2] received reconcile message
Sep 22 23:22:28.081 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(712), op: ExtentFlush { repair_id: ReconciliationId(712), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.081 INFO [2] client ExtentFlush { repair_id: ReconciliationId(712), extent_id: 120, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.082 DEBG 712 Flush extent 120 with f:2 g:2
Sep 22 23:22:28.082 DEBG Flush just extent 120 with f:2 and g:2
Sep 22 23:22:28.082 DEBG [1] It's time to notify for 712
Sep 22 23:22:28.082 INFO Completion from [1] id:712 status:true
Sep 22 23:22:28.082 INFO [713/752] Repair commands completed
Sep 22 23:22:28.082 INFO Pop front: ReconcileIO { id: ReconciliationId(713), op: ExtentClose { repair_id: ReconciliationId(713), extent_id: 120 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.082 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.082 INFO [0] received reconcile message
Sep 22 23:22:28.082 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(713), op: ExtentClose { repair_id: ReconciliationId(713), extent_id: 120 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.082 INFO [0] client ExtentClose { repair_id: ReconciliationId(713), extent_id: 120 }
Sep 22 23:22:28.082 INFO [1] received reconcile message
Sep 22 23:22:28.082 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(713), op: ExtentClose { repair_id: ReconciliationId(713), extent_id: 120 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.082 INFO [1] client ExtentClose { repair_id: ReconciliationId(713), extent_id: 120 }
Sep 22 23:22:28.082 INFO [2] received reconcile message
Sep 22 23:22:28.082 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(713), op: ExtentClose { repair_id: ReconciliationId(713), extent_id: 120 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.082 INFO [2] client ExtentClose { repair_id: ReconciliationId(713), extent_id: 120 }
Sep 22 23:22:28.082 DEBG 713 Close extent 120
Sep 22 23:22:28.082 DEBG 713 Close extent 120
Sep 22 23:22:28.083 DEBG 713 Close extent 120
Sep 22 23:22:28.083 DEBG [2] It's time to notify for 713
Sep 22 23:22:28.083 INFO Completion from [2] id:713 status:true
Sep 22 23:22:28.083 INFO [714/752] Repair commands completed
Sep 22 23:22:28.083 INFO Pop front: ReconcileIO { id: ReconciliationId(714), op: ExtentRepair { repair_id: ReconciliationId(714), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.083 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.083 INFO [0] received reconcile message
Sep 22 23:22:28.083 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(714), op: ExtentRepair { repair_id: ReconciliationId(714), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.083 INFO [0] client ExtentRepair { repair_id: ReconciliationId(714), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.083 INFO [0] Sending repair request ReconciliationId(714)
Sep 22 23:22:28.083 INFO [1] received reconcile message
Sep 22 23:22:28.083 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(714), op: ExtentRepair { repair_id: ReconciliationId(714), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.084 INFO [1] client ExtentRepair { repair_id: ReconciliationId(714), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.084 INFO [1] No action required ReconciliationId(714)
Sep 22 23:22:28.084 INFO [2] received reconcile message
Sep 22 23:22:28.084 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(714), op: ExtentRepair { repair_id: ReconciliationId(714), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.084 INFO [2] client ExtentRepair { repair_id: ReconciliationId(714), extent_id: 120, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.084 INFO [2] No action required ReconciliationId(714)
Sep 22 23:22:28.084 DEBG 714 Repair extent 120 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.084 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/078.copy"
Sep 22 23:22:28.147 INFO accepted connection, remote_addr: 127.0.0.1:59646, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.147 TRCE incoming request, uri: /extent/120/files, method: GET, req_id: 4b80a61b-e7d9-4ed1-ad54-4e3780892680, remote_addr: 127.0.0.1:59646, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.147 INFO request completed, latency_us: 232, response_code: 200, uri: /extent/120/files, method: GET, req_id: 4b80a61b-e7d9-4ed1-ad54-4e3780892680, remote_addr: 127.0.0.1:59646, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.148 INFO eid:120 Found repair files: ["078", "078.db"]
Sep 22 23:22:28.148 TRCE incoming request, uri: /newextent/120/data, method: GET, req_id: e988c35e-260e-40a1-ae9f-bc0eeb5735eb, remote_addr: 127.0.0.1:59646, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.148 INFO request completed, latency_us: 352, response_code: 200, uri: /newextent/120/data, method: GET, req_id: e988c35e-260e-40a1-ae9f-bc0eeb5735eb, remote_addr: 127.0.0.1:59646, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.151 DEBG up_ds_listen was notified
Sep 22 23:22:28.151 DEBG up_ds_listen process 1092
Sep 22 23:22:28.151 DEBG [A] ack job 1092:93, : downstairs
Sep 22 23:22:28.151 DEBG up_ds_listen process 1093
Sep 22 23:22:28.151 DEBG [A] ack job 1093:94, : downstairs
Sep 22 23:22:28.153 TRCE incoming request, uri: /newextent/120/db, method: GET, req_id: 18777097-13d1-4ecd-a9eb-cad5db9563f6, remote_addr: 127.0.0.1:59646, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.154 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/120/db, method: GET, req_id: 18777097-13d1-4ecd-a9eb-cad5db9563f6, remote_addr: 127.0.0.1:59646, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.155 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/078.copy" to "/tmp/downstairs-zrMnlo6G/00/000/078.replace"
Sep 22 23:22:28.155 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.156 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/078.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.156 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/078"
Sep 22 23:22:28.156 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/078.db"
Sep 22 23:22:28.156 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.156 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/078.replace" to "/tmp/downstairs-zrMnlo6G/00/000/078.completed"
Sep 22 23:22:28.156 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.156 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.157 DEBG [0] It's time to notify for 714
Sep 22 23:22:28.157 INFO Completion from [0] id:714 status:true
Sep 22 23:22:28.157 INFO [715/752] Repair commands completed
Sep 22 23:22:28.157 INFO Pop front: ReconcileIO { id: ReconciliationId(715), op: ExtentReopen { repair_id: ReconciliationId(715), extent_id: 120 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.157 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.157 INFO [0] received reconcile message
Sep 22 23:22:28.157 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(715), op: ExtentReopen { repair_id: ReconciliationId(715), extent_id: 120 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.157 INFO [0] client ExtentReopen { repair_id: ReconciliationId(715), extent_id: 120 }
Sep 22 23:22:28.157 INFO [1] received reconcile message
Sep 22 23:22:28.157 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(715), op: ExtentReopen { repair_id: ReconciliationId(715), extent_id: 120 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.157 INFO [1] client ExtentReopen { repair_id: ReconciliationId(715), extent_id: 120 }
Sep 22 23:22:28.157 INFO [2] received reconcile message
Sep 22 23:22:28.157 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(715), op: ExtentReopen { repair_id: ReconciliationId(715), extent_id: 120 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.157 INFO [2] client ExtentReopen { repair_id: ReconciliationId(715), extent_id: 120 }
Sep 22 23:22:28.157 DEBG 715 Reopen extent 120
Sep 22 23:22:28.158 DEBG 715 Reopen extent 120
Sep 22 23:22:28.158 DEBG 715 Reopen extent 120
Sep 22 23:22:28.159 DEBG [2] It's time to notify for 715
Sep 22 23:22:28.159 INFO Completion from [2] id:715 status:true
Sep 22 23:22:28.159 INFO [716/752] Repair commands completed
Sep 22 23:22:28.159 INFO Pop front: ReconcileIO { id: ReconciliationId(716), op: ExtentFlush { repair_id: ReconciliationId(716), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.159 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.159 INFO [0] received reconcile message
Sep 22 23:22:28.159 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(716), op: ExtentFlush { repair_id: ReconciliationId(716), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.159 INFO [0] client ExtentFlush { repair_id: ReconciliationId(716), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.159 INFO [1] received reconcile message
Sep 22 23:22:28.159 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(716), op: ExtentFlush { repair_id: ReconciliationId(716), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.159 INFO [1] client ExtentFlush { repair_id: ReconciliationId(716), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.159 INFO [2] received reconcile message
Sep 22 23:22:28.159 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(716), op: ExtentFlush { repair_id: ReconciliationId(716), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.159 INFO [2] client ExtentFlush { repair_id: ReconciliationId(716), extent_id: 145, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.160 DEBG 716 Flush extent 145 with f:2 g:2
Sep 22 23:22:28.160 DEBG Flush just extent 145 with f:2 and g:2
Sep 22 23:22:28.160 DEBG [1] It's time to notify for 716
Sep 22 23:22:28.160 INFO Completion from [1] id:716 status:true
Sep 22 23:22:28.160 INFO [717/752] Repair commands completed
Sep 22 23:22:28.160 INFO Pop front: ReconcileIO { id: ReconciliationId(717), op: ExtentClose { repair_id: ReconciliationId(717), extent_id: 145 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.160 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.160 INFO [0] received reconcile message
Sep 22 23:22:28.160 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(717), op: ExtentClose { repair_id: ReconciliationId(717), extent_id: 145 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.160 INFO [0] client ExtentClose { repair_id: ReconciliationId(717), extent_id: 145 }
Sep 22 23:22:28.160 INFO [1] received reconcile message
Sep 22 23:22:28.160 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(717), op: ExtentClose { repair_id: ReconciliationId(717), extent_id: 145 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.160 INFO [1] client ExtentClose { repair_id: ReconciliationId(717), extent_id: 145 }
Sep 22 23:22:28.160 INFO [2] received reconcile message
Sep 22 23:22:28.160 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(717), op: ExtentClose { repair_id: ReconciliationId(717), extent_id: 145 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.160 INFO [2] client ExtentClose { repair_id: ReconciliationId(717), extent_id: 145 }
Sep 22 23:22:28.160 DEBG 717 Close extent 145
Sep 22 23:22:28.160 DEBG 717 Close extent 145
Sep 22 23:22:28.161 DEBG 717 Close extent 145
Sep 22 23:22:28.161 DEBG [2] It's time to notify for 717
Sep 22 23:22:28.161 INFO Completion from [2] id:717 status:true
Sep 22 23:22:28.161 INFO [718/752] Repair commands completed
Sep 22 23:22:28.161 INFO Pop front: ReconcileIO { id: ReconciliationId(718), op: ExtentRepair { repair_id: ReconciliationId(718), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.161 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.161 INFO [0] received reconcile message
Sep 22 23:22:28.161 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(718), op: ExtentRepair { repair_id: ReconciliationId(718), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.161 INFO [0] client ExtentRepair { repair_id: ReconciliationId(718), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.161 INFO [0] Sending repair request ReconciliationId(718)
Sep 22 23:22:28.161 INFO [1] received reconcile message
Sep 22 23:22:28.161 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(718), op: ExtentRepair { repair_id: ReconciliationId(718), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.161 INFO [1] client ExtentRepair { repair_id: ReconciliationId(718), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.161 INFO [1] No action required ReconciliationId(718)
Sep 22 23:22:28.162 INFO [2] received reconcile message
Sep 22 23:22:28.162 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(718), op: ExtentRepair { repair_id: ReconciliationId(718), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.162 INFO [2] client ExtentRepair { repair_id: ReconciliationId(718), extent_id: 145, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.162 INFO [2] No action required ReconciliationId(718)
Sep 22 23:22:28.162 DEBG 718 Repair extent 145 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.162 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/091.copy"
Sep 22 23:22:28.204 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:22:28.204 DEBG up_ds_listen was notified
Sep 22 23:22:28.204 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:22:28.205 DEBG Flush     :1092 extent_limit None deps:[JobId(1091), JobId(1090)] res:true f:35 g:1
Sep 22 23:22:28.211 DEBG Read      :1093 deps:[JobId(1092)] res:true
Sep 22 23:22:28.227 INFO accepted connection, remote_addr: 127.0.0.1:34697, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.228 TRCE incoming request, uri: /extent/145/files, method: GET, req_id: 365847b7-a027-48dd-a7f1-8f480e1cc27a, remote_addr: 127.0.0.1:34697, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.228 INFO request completed, latency_us: 239, response_code: 200, uri: /extent/145/files, method: GET, req_id: 365847b7-a027-48dd-a7f1-8f480e1cc27a, remote_addr: 127.0.0.1:34697, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.228 INFO eid:145 Found repair files: ["091", "091.db"]
Sep 22 23:22:28.228 TRCE incoming request, uri: /newextent/145/data, method: GET, req_id: 370067db-4190-4d4b-89c5-f2f9970c46cf, remote_addr: 127.0.0.1:34697, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.229 INFO request completed, latency_us: 349, response_code: 200, uri: /newextent/145/data, method: GET, req_id: 370067db-4190-4d4b-89c5-f2f9970c46cf, remote_addr: 127.0.0.1:34697, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.233 DEBG IO Read  1095 has deps [JobId(1094)]
Sep 22 23:22:28.233 DEBG [rc] retire 1092 clears [JobId(1091), JobId(1092)], : downstairs
Sep 22 23:22:28.234 TRCE incoming request, uri: /newextent/145/db, method: GET, req_id: bba38001-333e-4cdd-8004-746ece71293f, remote_addr: 127.0.0.1:34697, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.234 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/145/db, method: GET, req_id: bba38001-333e-4cdd-8004-746ece71293f, remote_addr: 127.0.0.1:34697, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.235 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/091.copy" to "/tmp/downstairs-zrMnlo6G/00/000/091.replace"
Sep 22 23:22:28.235 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.236 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/091.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.236 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/091"
Sep 22 23:22:28.236 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/091.db"
Sep 22 23:22:28.236 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.236 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/091.replace" to "/tmp/downstairs-zrMnlo6G/00/000/091.completed"
Sep 22 23:22:28.236 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.237 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.237 DEBG [0] It's time to notify for 718
Sep 22 23:22:28.237 INFO Completion from [0] id:718 status:true
Sep 22 23:22:28.237 INFO [719/752] Repair commands completed
Sep 22 23:22:28.237 INFO Pop front: ReconcileIO { id: ReconciliationId(719), op: ExtentReopen { repair_id: ReconciliationId(719), extent_id: 145 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.237 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.237 INFO [0] received reconcile message
Sep 22 23:22:28.237 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(719), op: ExtentReopen { repair_id: ReconciliationId(719), extent_id: 145 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.237 INFO [0] client ExtentReopen { repair_id: ReconciliationId(719), extent_id: 145 }
Sep 22 23:22:28.237 INFO [1] received reconcile message
Sep 22 23:22:28.237 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(719), op: ExtentReopen { repair_id: ReconciliationId(719), extent_id: 145 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.237 INFO [1] client ExtentReopen { repair_id: ReconciliationId(719), extent_id: 145 }
Sep 22 23:22:28.237 INFO [2] received reconcile message
Sep 22 23:22:28.237 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(719), op: ExtentReopen { repair_id: ReconciliationId(719), extent_id: 145 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.237 INFO [2] client ExtentReopen { repair_id: ReconciliationId(719), extent_id: 145 }
Sep 22 23:22:28.237 DEBG 719 Reopen extent 145
Sep 22 23:22:28.238 DEBG 719 Reopen extent 145
Sep 22 23:22:28.239 DEBG 719 Reopen extent 145
Sep 22 23:22:28.239 DEBG [2] It's time to notify for 719
Sep 22 23:22:28.239 INFO Completion from [2] id:719 status:true
Sep 22 23:22:28.239 INFO [720/752] Repair commands completed
Sep 22 23:22:28.239 INFO Pop front: ReconcileIO { id: ReconciliationId(720), op: ExtentFlush { repair_id: ReconciliationId(720), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.239 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.239 INFO [0] received reconcile message
Sep 22 23:22:28.239 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(720), op: ExtentFlush { repair_id: ReconciliationId(720), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.239 INFO [0] client ExtentFlush { repair_id: ReconciliationId(720), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.239 INFO [1] received reconcile message
Sep 22 23:22:28.239 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(720), op: ExtentFlush { repair_id: ReconciliationId(720), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.239 INFO [1] client ExtentFlush { repair_id: ReconciliationId(720), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.239 INFO [2] received reconcile message
Sep 22 23:22:28.239 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(720), op: ExtentFlush { repair_id: ReconciliationId(720), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.239 INFO [2] client ExtentFlush { repair_id: ReconciliationId(720), extent_id: 131, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.240 DEBG 720 Flush extent 131 with f:2 g:2
Sep 22 23:22:28.240 DEBG Flush just extent 131 with f:2 and g:2
Sep 22 23:22:28.240 DEBG [1] It's time to notify for 720
Sep 22 23:22:28.240 INFO Completion from [1] id:720 status:true
Sep 22 23:22:28.240 INFO [721/752] Repair commands completed
Sep 22 23:22:28.240 INFO Pop front: ReconcileIO { id: ReconciliationId(721), op: ExtentClose { repair_id: ReconciliationId(721), extent_id: 131 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.240 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.240 INFO [0] received reconcile message
Sep 22 23:22:28.240 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(721), op: ExtentClose { repair_id: ReconciliationId(721), extent_id: 131 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.240 INFO [0] client ExtentClose { repair_id: ReconciliationId(721), extent_id: 131 }
Sep 22 23:22:28.240 INFO [1] received reconcile message
Sep 22 23:22:28.240 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(721), op: ExtentClose { repair_id: ReconciliationId(721), extent_id: 131 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.240 INFO [1] client ExtentClose { repair_id: ReconciliationId(721), extent_id: 131 }
Sep 22 23:22:28.240 INFO [2] received reconcile message
Sep 22 23:22:28.240 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(721), op: ExtentClose { repair_id: ReconciliationId(721), extent_id: 131 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.240 INFO [2] client ExtentClose { repair_id: ReconciliationId(721), extent_id: 131 }
Sep 22 23:22:28.240 DEBG 721 Close extent 131
Sep 22 23:22:28.240 DEBG 721 Close extent 131
Sep 22 23:22:28.241 DEBG 721 Close extent 131
Sep 22 23:22:28.241 DEBG [2] It's time to notify for 721
Sep 22 23:22:28.241 INFO Completion from [2] id:721 status:true
Sep 22 23:22:28.241 INFO [722/752] Repair commands completed
Sep 22 23:22:28.241 INFO Pop front: ReconcileIO { id: ReconciliationId(722), op: ExtentRepair { repair_id: ReconciliationId(722), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.241 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.241 INFO [0] received reconcile message
Sep 22 23:22:28.241 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(722), op: ExtentRepair { repair_id: ReconciliationId(722), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.241 INFO [0] client ExtentRepair { repair_id: ReconciliationId(722), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.241 INFO [0] Sending repair request ReconciliationId(722)
Sep 22 23:22:28.241 INFO [1] received reconcile message
Sep 22 23:22:28.241 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(722), op: ExtentRepair { repair_id: ReconciliationId(722), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.241 INFO [1] client ExtentRepair { repair_id: ReconciliationId(722), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.241 INFO [1] No action required ReconciliationId(722)
Sep 22 23:22:28.242 INFO [2] received reconcile message
Sep 22 23:22:28.242 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(722), op: ExtentRepair { repair_id: ReconciliationId(722), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.242 INFO [2] client ExtentRepair { repair_id: ReconciliationId(722), extent_id: 131, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.242 INFO [2] No action required ReconciliationId(722)
Sep 22 23:22:28.242 DEBG 722 Repair extent 131 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.242 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/083.copy"
Sep 22 23:22:28.248 WARN returning error on flush!
Sep 22 23:22:28.248 DEBG Flush     :1094 extent_limit None deps:[JobId(1093), JobId(1092)] res:false f:36 g:1
Sep 22 23:22:28.248 DEBG Flush     :1094 extent_limit None deps:[JobId(1093), JobId(1092)] res:true f:36 g:1
Sep 22 23:22:28.253 DEBG Read      :1095 deps:[JobId(1094)] res:true
Sep 22 23:22:28.304 INFO accepted connection, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.304 TRCE incoming request, uri: /extent/131/files, method: GET, req_id: 3ffacf74-8166-47f0-98fa-a83792402b1c, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.304 INFO request completed, latency_us: 225, response_code: 200, uri: /extent/131/files, method: GET, req_id: 3ffacf74-8166-47f0-98fa-a83792402b1c, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.305 INFO eid:131 Found repair files: ["083", "083.db"]
Sep 22 23:22:28.305 TRCE incoming request, uri: /newextent/131/data, method: GET, req_id: 89149804-e68d-43f9-a76d-c96f7af34546, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.305 INFO request completed, latency_us: 340, response_code: 200, uri: /newextent/131/data, method: GET, req_id: 89149804-e68d-43f9-a76d-c96f7af34546, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.310 TRCE incoming request, uri: /newextent/131/db, method: GET, req_id: a7da5631-e13e-43a7-a454-6f3e4a62d8a6, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.310 INFO request completed, latency_us: 300, response_code: 200, uri: /newextent/131/db, method: GET, req_id: a7da5631-e13e-43a7-a454-6f3e4a62d8a6, remote_addr: 127.0.0.1:46641, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.311 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/083.copy" to "/tmp/downstairs-zrMnlo6G/00/000/083.replace"
Sep 22 23:22:28.312 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.312 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/083.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.313 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/083"
Sep 22 23:22:28.313 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/083.db"
Sep 22 23:22:28.313 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.313 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/083.replace" to "/tmp/downstairs-zrMnlo6G/00/000/083.completed"
Sep 22 23:22:28.313 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.313 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.313 DEBG [0] It's time to notify for 722
Sep 22 23:22:28.313 INFO Completion from [0] id:722 status:true
Sep 22 23:22:28.313 INFO [723/752] Repair commands completed
Sep 22 23:22:28.313 INFO Pop front: ReconcileIO { id: ReconciliationId(723), op: ExtentReopen { repair_id: ReconciliationId(723), extent_id: 131 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.313 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.313 INFO [0] received reconcile message
Sep 22 23:22:28.313 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(723), op: ExtentReopen { repair_id: ReconciliationId(723), extent_id: 131 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.313 INFO [0] client ExtentReopen { repair_id: ReconciliationId(723), extent_id: 131 }
Sep 22 23:22:28.313 INFO [1] received reconcile message
Sep 22 23:22:28.313 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(723), op: ExtentReopen { repair_id: ReconciliationId(723), extent_id: 131 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.314 INFO [1] client ExtentReopen { repair_id: ReconciliationId(723), extent_id: 131 }
Sep 22 23:22:28.314 INFO [2] received reconcile message
Sep 22 23:22:28.314 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(723), op: ExtentReopen { repair_id: ReconciliationId(723), extent_id: 131 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.314 INFO [2] client ExtentReopen { repair_id: ReconciliationId(723), extent_id: 131 }
Sep 22 23:22:28.314 DEBG 723 Reopen extent 131
Sep 22 23:22:28.314 DEBG 723 Reopen extent 131
Sep 22 23:22:28.315 DEBG 723 Reopen extent 131
Sep 22 23:22:28.316 DEBG [2] It's time to notify for 723
Sep 22 23:22:28.316 INFO Completion from [2] id:723 status:true
Sep 22 23:22:28.316 INFO [724/752] Repair commands completed
Sep 22 23:22:28.316 INFO Pop front: ReconcileIO { id: ReconciliationId(724), op: ExtentFlush { repair_id: ReconciliationId(724), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.316 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.316 INFO [0] received reconcile message
Sep 22 23:22:28.316 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(724), op: ExtentFlush { repair_id: ReconciliationId(724), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.316 INFO [0] client ExtentFlush { repair_id: ReconciliationId(724), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.316 INFO [1] received reconcile message
Sep 22 23:22:28.316 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(724), op: ExtentFlush { repair_id: ReconciliationId(724), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.316 INFO [1] client ExtentFlush { repair_id: ReconciliationId(724), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.316 INFO [2] received reconcile message
Sep 22 23:22:28.316 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(724), op: ExtentFlush { repair_id: ReconciliationId(724), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.316 INFO [2] client ExtentFlush { repair_id: ReconciliationId(724), extent_id: 170, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.316 DEBG 724 Flush extent 170 with f:2 g:2
Sep 22 23:22:28.316 DEBG Flush just extent 170 with f:2 and g:2
Sep 22 23:22:28.316 DEBG [1] It's time to notify for 724
Sep 22 23:22:28.316 INFO Completion from [1] id:724 status:true
Sep 22 23:22:28.316 INFO [725/752] Repair commands completed
Sep 22 23:22:28.316 INFO Pop front: ReconcileIO { id: ReconciliationId(725), op: ExtentClose { repair_id: ReconciliationId(725), extent_id: 170 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.316 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.316 INFO [0] received reconcile message
Sep 22 23:22:28.316 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(725), op: ExtentClose { repair_id: ReconciliationId(725), extent_id: 170 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.316 INFO [0] client ExtentClose { repair_id: ReconciliationId(725), extent_id: 170 }
Sep 22 23:22:28.316 INFO [1] received reconcile message
Sep 22 23:22:28.316 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(725), op: ExtentClose { repair_id: ReconciliationId(725), extent_id: 170 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.316 INFO [1] client ExtentClose { repair_id: ReconciliationId(725), extent_id: 170 }
Sep 22 23:22:28.316 INFO [2] received reconcile message
Sep 22 23:22:28.316 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(725), op: ExtentClose { repair_id: ReconciliationId(725), extent_id: 170 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.316 INFO [2] client ExtentClose { repair_id: ReconciliationId(725), extent_id: 170 }
Sep 22 23:22:28.317 DEBG 725 Close extent 170
Sep 22 23:22:28.317 DEBG 725 Close extent 170
Sep 22 23:22:28.317 DEBG 725 Close extent 170
Sep 22 23:22:28.318 DEBG [2] It's time to notify for 725
Sep 22 23:22:28.318 INFO Completion from [2] id:725 status:true
Sep 22 23:22:28.318 INFO [726/752] Repair commands completed
Sep 22 23:22:28.318 INFO Pop front: ReconcileIO { id: ReconciliationId(726), op: ExtentRepair { repair_id: ReconciliationId(726), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.318 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.318 INFO [0] received reconcile message
Sep 22 23:22:28.318 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(726), op: ExtentRepair { repair_id: ReconciliationId(726), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.318 INFO [0] client ExtentRepair { repair_id: ReconciliationId(726), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.318 INFO [0] Sending repair request ReconciliationId(726)
Sep 22 23:22:28.318 INFO [1] received reconcile message
Sep 22 23:22:28.318 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(726), op: ExtentRepair { repair_id: ReconciliationId(726), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.318 INFO [1] client ExtentRepair { repair_id: ReconciliationId(726), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.318 INFO [1] No action required ReconciliationId(726)
Sep 22 23:22:28.318 INFO [2] received reconcile message
Sep 22 23:22:28.318 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(726), op: ExtentRepair { repair_id: ReconciliationId(726), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.318 INFO [2] client ExtentRepair { repair_id: ReconciliationId(726), extent_id: 170, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.318 INFO [2] No action required ReconciliationId(726)
Sep 22 23:22:28.318 DEBG 726 Repair extent 170 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.318 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0AA.copy"
Sep 22 23:22:28.384 INFO accepted connection, remote_addr: 127.0.0.1:40005, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.384 TRCE incoming request, uri: /extent/170/files, method: GET, req_id: 968623e9-6482-42b6-bf97-f826537761f4, remote_addr: 127.0.0.1:40005, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.385 INFO request completed, latency_us: 234, response_code: 200, uri: /extent/170/files, method: GET, req_id: 968623e9-6482-42b6-bf97-f826537761f4, remote_addr: 127.0.0.1:40005, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.385 INFO eid:170 Found repair files: ["0AA", "0AA.db"]
Sep 22 23:22:28.385 TRCE incoming request, uri: /newextent/170/data, method: GET, req_id: 36d0443d-aab3-4ac4-a914-1ac7588303f3, remote_addr: 127.0.0.1:40005, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.386 INFO request completed, latency_us: 329, response_code: 200, uri: /newextent/170/data, method: GET, req_id: 36d0443d-aab3-4ac4-a914-1ac7588303f3, remote_addr: 127.0.0.1:40005, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.390 TRCE incoming request, uri: /newextent/170/db, method: GET, req_id: 51129285-e60f-4194-8722-c84978748b65, remote_addr: 127.0.0.1:40005, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.391 INFO request completed, latency_us: 304, response_code: 200, uri: /newextent/170/db, method: GET, req_id: 51129285-e60f-4194-8722-c84978748b65, remote_addr: 127.0.0.1:40005, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.392 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0AA.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0AA.replace"
Sep 22 23:22:28.392 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.393 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0AA.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.393 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AA"
Sep 22 23:22:28.393 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0AA.db"
Sep 22 23:22:28.393 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.393 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0AA.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0AA.completed"
Sep 22 23:22:28.393 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.393 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.393 DEBG [0] It's time to notify for 726
Sep 22 23:22:28.393 INFO Completion from [0] id:726 status:true
Sep 22 23:22:28.393 INFO [727/752] Repair commands completed
Sep 22 23:22:28.393 INFO Pop front: ReconcileIO { id: ReconciliationId(727), op: ExtentReopen { repair_id: ReconciliationId(727), extent_id: 170 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.393 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.394 INFO [0] received reconcile message
Sep 22 23:22:28.394 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(727), op: ExtentReopen { repair_id: ReconciliationId(727), extent_id: 170 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.394 INFO [0] client ExtentReopen { repair_id: ReconciliationId(727), extent_id: 170 }
Sep 22 23:22:28.394 INFO [1] received reconcile message
Sep 22 23:22:28.394 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(727), op: ExtentReopen { repair_id: ReconciliationId(727), extent_id: 170 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.394 INFO [1] client ExtentReopen { repair_id: ReconciliationId(727), extent_id: 170 }
Sep 22 23:22:28.394 INFO [2] received reconcile message
Sep 22 23:22:28.394 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(727), op: ExtentReopen { repair_id: ReconciliationId(727), extent_id: 170 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.394 INFO [2] client ExtentReopen { repair_id: ReconciliationId(727), extent_id: 170 }
Sep 22 23:22:28.394 DEBG 727 Reopen extent 170
Sep 22 23:22:28.395 DEBG 727 Reopen extent 170
Sep 22 23:22:28.395 DEBG 727 Reopen extent 170
Sep 22 23:22:28.396 DEBG [2] It's time to notify for 727
Sep 22 23:22:28.396 INFO Completion from [2] id:727 status:true
Sep 22 23:22:28.396 INFO [728/752] Repair commands completed
Sep 22 23:22:28.396 INFO Pop front: ReconcileIO { id: ReconciliationId(728), op: ExtentFlush { repair_id: ReconciliationId(728), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.396 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.396 INFO [0] received reconcile message
Sep 22 23:22:28.396 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(728), op: ExtentFlush { repair_id: ReconciliationId(728), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.396 INFO [0] client ExtentFlush { repair_id: ReconciliationId(728), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.396 INFO [1] received reconcile message
Sep 22 23:22:28.396 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(728), op: ExtentFlush { repair_id: ReconciliationId(728), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.396 INFO [1] client ExtentFlush { repair_id: ReconciliationId(728), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.396 INFO [2] received reconcile message
Sep 22 23:22:28.396 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(728), op: ExtentFlush { repair_id: ReconciliationId(728), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.396 INFO [2] client ExtentFlush { repair_id: ReconciliationId(728), extent_id: 72, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.396 DEBG 728 Flush extent 72 with f:2 g:2
Sep 22 23:22:28.396 DEBG Flush just extent 72 with f:2 and g:2
Sep 22 23:22:28.396 DEBG [1] It's time to notify for 728
Sep 22 23:22:28.396 INFO Completion from [1] id:728 status:true
Sep 22 23:22:28.396 INFO [729/752] Repair commands completed
Sep 22 23:22:28.396 INFO Pop front: ReconcileIO { id: ReconciliationId(729), op: ExtentClose { repair_id: ReconciliationId(729), extent_id: 72 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.396 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.396 INFO [0] received reconcile message
Sep 22 23:22:28.396 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(729), op: ExtentClose { repair_id: ReconciliationId(729), extent_id: 72 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.396 INFO [0] client ExtentClose { repair_id: ReconciliationId(729), extent_id: 72 }
Sep 22 23:22:28.397 INFO [1] received reconcile message
Sep 22 23:22:28.397 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(729), op: ExtentClose { repair_id: ReconciliationId(729), extent_id: 72 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.397 INFO [1] client ExtentClose { repair_id: ReconciliationId(729), extent_id: 72 }
Sep 22 23:22:28.397 INFO [2] received reconcile message
Sep 22 23:22:28.397 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(729), op: ExtentClose { repair_id: ReconciliationId(729), extent_id: 72 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.397 INFO [2] client ExtentClose { repair_id: ReconciliationId(729), extent_id: 72 }
Sep 22 23:22:28.397 DEBG 729 Close extent 72
Sep 22 23:22:28.397 DEBG 729 Close extent 72
Sep 22 23:22:28.397 DEBG 729 Close extent 72
Sep 22 23:22:28.398 DEBG [2] It's time to notify for 729
Sep 22 23:22:28.398 INFO Completion from [2] id:729 status:true
Sep 22 23:22:28.398 INFO [730/752] Repair commands completed
Sep 22 23:22:28.398 INFO Pop front: ReconcileIO { id: ReconciliationId(730), op: ExtentRepair { repair_id: ReconciliationId(730), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.398 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.398 INFO [0] received reconcile message
Sep 22 23:22:28.398 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(730), op: ExtentRepair { repair_id: ReconciliationId(730), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.398 INFO [0] client ExtentRepair { repair_id: ReconciliationId(730), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.398 INFO [0] Sending repair request ReconciliationId(730)
Sep 22 23:22:28.398 INFO [1] received reconcile message
Sep 22 23:22:28.398 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(730), op: ExtentRepair { repair_id: ReconciliationId(730), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.398 INFO [1] client ExtentRepair { repair_id: ReconciliationId(730), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.398 INFO [1] No action required ReconciliationId(730)
Sep 22 23:22:28.398 INFO [2] received reconcile message
Sep 22 23:22:28.398 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(730), op: ExtentRepair { repair_id: ReconciliationId(730), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.398 INFO [2] client ExtentRepair { repair_id: ReconciliationId(730), extent_id: 72, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.398 INFO [2] No action required ReconciliationId(730)
Sep 22 23:22:28.398 DEBG 730 Repair extent 72 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.398 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/048.copy"
Sep 22 23:22:28.464 INFO accepted connection, remote_addr: 127.0.0.1:45975, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.465 TRCE incoming request, uri: /extent/72/files, method: GET, req_id: 5e218426-d861-4c71-a05d-fce3b9e901fe, remote_addr: 127.0.0.1:45975, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.465 INFO request completed, latency_us: 245, response_code: 200, uri: /extent/72/files, method: GET, req_id: 5e218426-d861-4c71-a05d-fce3b9e901fe, remote_addr: 127.0.0.1:45975, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.465 INFO eid:72 Found repair files: ["048", "048.db"]
Sep 22 23:22:28.466 TRCE incoming request, uri: /newextent/72/data, method: GET, req_id: 4c86aad1-8539-4fb3-8971-494813115ec5, remote_addr: 127.0.0.1:45975, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.466 INFO request completed, latency_us: 273, response_code: 200, uri: /newextent/72/data, method: GET, req_id: 4c86aad1-8539-4fb3-8971-494813115ec5, remote_addr: 127.0.0.1:45975, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.471 TRCE incoming request, uri: /newextent/72/db, method: GET, req_id: 3e190832-c8ec-4c54-b936-06fdc1a5249f, remote_addr: 127.0.0.1:45975, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.471 INFO request completed, latency_us: 305, response_code: 200, uri: /newextent/72/db, method: GET, req_id: 3e190832-c8ec-4c54-b936-06fdc1a5249f, remote_addr: 127.0.0.1:45975, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.472 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/048.copy" to "/tmp/downstairs-zrMnlo6G/00/000/048.replace"
Sep 22 23:22:28.472 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.473 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/048.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.473 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/048"
Sep 22 23:22:28.473 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/048.db"
Sep 22 23:22:28.474 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.474 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/048.replace" to "/tmp/downstairs-zrMnlo6G/00/000/048.completed"
Sep 22 23:22:28.474 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.474 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.474 DEBG [0] It's time to notify for 730
Sep 22 23:22:28.474 INFO Completion from [0] id:730 status:true
Sep 22 23:22:28.474 INFO [731/752] Repair commands completed
Sep 22 23:22:28.474 INFO Pop front: ReconcileIO { id: ReconciliationId(731), op: ExtentReopen { repair_id: ReconciliationId(731), extent_id: 72 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.474 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.474 INFO [0] received reconcile message
Sep 22 23:22:28.474 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(731), op: ExtentReopen { repair_id: ReconciliationId(731), extent_id: 72 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.474 INFO [0] client ExtentReopen { repair_id: ReconciliationId(731), extent_id: 72 }
Sep 22 23:22:28.474 INFO [1] received reconcile message
Sep 22 23:22:28.474 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(731), op: ExtentReopen { repair_id: ReconciliationId(731), extent_id: 72 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.474 INFO [1] client ExtentReopen { repair_id: ReconciliationId(731), extent_id: 72 }
Sep 22 23:22:28.474 INFO [2] received reconcile message
Sep 22 23:22:28.474 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(731), op: ExtentReopen { repair_id: ReconciliationId(731), extent_id: 72 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.474 INFO [2] client ExtentReopen { repair_id: ReconciliationId(731), extent_id: 72 }
Sep 22 23:22:28.474 DEBG 731 Reopen extent 72
Sep 22 23:22:28.475 DEBG 731 Reopen extent 72
Sep 22 23:22:28.476 DEBG 731 Reopen extent 72
Sep 22 23:22:28.476 DEBG [2] It's time to notify for 731
Sep 22 23:22:28.476 INFO Completion from [2] id:731 status:true
Sep 22 23:22:28.476 INFO [732/752] Repair commands completed
Sep 22 23:22:28.476 INFO Pop front: ReconcileIO { id: ReconciliationId(732), op: ExtentFlush { repair_id: ReconciliationId(732), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.476 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.477 INFO [0] received reconcile message
Sep 22 23:22:28.477 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(732), op: ExtentFlush { repair_id: ReconciliationId(732), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.477 INFO [0] client ExtentFlush { repair_id: ReconciliationId(732), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.477 INFO [1] received reconcile message
Sep 22 23:22:28.477 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(732), op: ExtentFlush { repair_id: ReconciliationId(732), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.477 INFO [1] client ExtentFlush { repair_id: ReconciliationId(732), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.477 INFO [2] received reconcile message
Sep 22 23:22:28.477 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(732), op: ExtentFlush { repair_id: ReconciliationId(732), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.477 INFO [2] client ExtentFlush { repair_id: ReconciliationId(732), extent_id: 180, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.477 DEBG 732 Flush extent 180 with f:2 g:2
Sep 22 23:22:28.477 DEBG Flush just extent 180 with f:2 and g:2
Sep 22 23:22:28.477 DEBG [1] It's time to notify for 732
Sep 22 23:22:28.477 INFO Completion from [1] id:732 status:true
Sep 22 23:22:28.477 INFO [733/752] Repair commands completed
Sep 22 23:22:28.477 INFO Pop front: ReconcileIO { id: ReconciliationId(733), op: ExtentClose { repair_id: ReconciliationId(733), extent_id: 180 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.477 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.477 INFO [0] received reconcile message
Sep 22 23:22:28.477 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(733), op: ExtentClose { repair_id: ReconciliationId(733), extent_id: 180 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.477 INFO [0] client ExtentClose { repair_id: ReconciliationId(733), extent_id: 180 }
Sep 22 23:22:28.477 INFO [1] received reconcile message
Sep 22 23:22:28.477 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(733), op: ExtentClose { repair_id: ReconciliationId(733), extent_id: 180 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.477 INFO [1] client ExtentClose { repair_id: ReconciliationId(733), extent_id: 180 }
Sep 22 23:22:28.477 INFO [2] received reconcile message
Sep 22 23:22:28.477 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(733), op: ExtentClose { repair_id: ReconciliationId(733), extent_id: 180 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.477 INFO [2] client ExtentClose { repair_id: ReconciliationId(733), extent_id: 180 }
Sep 22 23:22:28.477 DEBG 733 Close extent 180
Sep 22 23:22:28.478 DEBG 733 Close extent 180
Sep 22 23:22:28.478 DEBG 733 Close extent 180
Sep 22 23:22:28.478 DEBG [2] It's time to notify for 733
Sep 22 23:22:28.478 INFO Completion from [2] id:733 status:true
Sep 22 23:22:28.478 INFO [734/752] Repair commands completed
Sep 22 23:22:28.478 INFO Pop front: ReconcileIO { id: ReconciliationId(734), op: ExtentRepair { repair_id: ReconciliationId(734), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.479 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.479 INFO [0] received reconcile message
Sep 22 23:22:28.479 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(734), op: ExtentRepair { repair_id: ReconciliationId(734), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.479 INFO [0] client ExtentRepair { repair_id: ReconciliationId(734), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.479 INFO [0] Sending repair request ReconciliationId(734)
Sep 22 23:22:28.479 INFO [1] received reconcile message
Sep 22 23:22:28.479 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(734), op: ExtentRepair { repair_id: ReconciliationId(734), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.479 INFO [1] client ExtentRepair { repair_id: ReconciliationId(734), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.479 INFO [1] No action required ReconciliationId(734)
Sep 22 23:22:28.479 INFO [2] received reconcile message
Sep 22 23:22:28.479 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(734), op: ExtentRepair { repair_id: ReconciliationId(734), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.479 INFO [2] client ExtentRepair { repair_id: ReconciliationId(734), extent_id: 180, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.479 INFO [2] No action required ReconciliationId(734)
Sep 22 23:22:28.479 DEBG 734 Repair extent 180 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.479 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/0B4.copy"
Sep 22 23:22:28.543 INFO accepted connection, remote_addr: 127.0.0.1:46864, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.544 TRCE incoming request, uri: /extent/180/files, method: GET, req_id: 8c035ce6-8000-4998-b2f8-0a72fb2d67e4, remote_addr: 127.0.0.1:46864, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.544 INFO request completed, latency_us: 232, response_code: 200, uri: /extent/180/files, method: GET, req_id: 8c035ce6-8000-4998-b2f8-0a72fb2d67e4, remote_addr: 127.0.0.1:46864, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.544 INFO eid:180 Found repair files: ["0B4", "0B4.db"]
Sep 22 23:22:28.545 TRCE incoming request, uri: /newextent/180/data, method: GET, req_id: 86be73b5-5838-4eba-a10e-73d46abb33c9, remote_addr: 127.0.0.1:46864, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.545 INFO request completed, latency_us: 273, response_code: 200, uri: /newextent/180/data, method: GET, req_id: 86be73b5-5838-4eba-a10e-73d46abb33c9, remote_addr: 127.0.0.1:46864, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.550 TRCE incoming request, uri: /newextent/180/db, method: GET, req_id: a040e66d-c668-4bef-a3a2-19a7c98faf8c, remote_addr: 127.0.0.1:46864, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.550 INFO request completed, latency_us: 312, response_code: 200, uri: /newextent/180/db, method: GET, req_id: a040e66d-c668-4bef-a3a2-19a7c98faf8c, remote_addr: 127.0.0.1:46864, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.551 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/0B4.copy" to "/tmp/downstairs-zrMnlo6G/00/000/0B4.replace"
Sep 22 23:22:28.551 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.552 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/0B4.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.553 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B4"
Sep 22 23:22:28.553 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/0B4.db"
Sep 22 23:22:28.553 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.553 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/0B4.replace" to "/tmp/downstairs-zrMnlo6G/00/000/0B4.completed"
Sep 22 23:22:28.553 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.553 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.553 DEBG [0] It's time to notify for 734
Sep 22 23:22:28.553 INFO Completion from [0] id:734 status:true
Sep 22 23:22:28.553 INFO [735/752] Repair commands completed
Sep 22 23:22:28.553 INFO Pop front: ReconcileIO { id: ReconciliationId(735), op: ExtentReopen { repair_id: ReconciliationId(735), extent_id: 180 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.553 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.553 INFO [0] received reconcile message
Sep 22 23:22:28.553 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(735), op: ExtentReopen { repair_id: ReconciliationId(735), extent_id: 180 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.553 INFO [0] client ExtentReopen { repair_id: ReconciliationId(735), extent_id: 180 }
Sep 22 23:22:28.554 INFO [1] received reconcile message
Sep 22 23:22:28.554 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(735), op: ExtentReopen { repair_id: ReconciliationId(735), extent_id: 180 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.554 INFO [1] client ExtentReopen { repair_id: ReconciliationId(735), extent_id: 180 }
Sep 22 23:22:28.554 INFO [2] received reconcile message
Sep 22 23:22:28.554 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(735), op: ExtentReopen { repair_id: ReconciliationId(735), extent_id: 180 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.554 INFO [2] client ExtentReopen { repair_id: ReconciliationId(735), extent_id: 180 }
Sep 22 23:22:28.554 DEBG 735 Reopen extent 180
Sep 22 23:22:28.555 DEBG 735 Reopen extent 180
Sep 22 23:22:28.555 DEBG 735 Reopen extent 180
Sep 22 23:22:28.556 DEBG [2] It's time to notify for 735
Sep 22 23:22:28.556 INFO Completion from [2] id:735 status:true
Sep 22 23:22:28.556 INFO [736/752] Repair commands completed
Sep 22 23:22:28.556 INFO Pop front: ReconcileIO { id: ReconciliationId(736), op: ExtentFlush { repair_id: ReconciliationId(736), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.556 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.556 INFO [0] received reconcile message
Sep 22 23:22:28.556 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(736), op: ExtentFlush { repair_id: ReconciliationId(736), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.556 INFO [0] client ExtentFlush { repair_id: ReconciliationId(736), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.556 INFO [1] received reconcile message
Sep 22 23:22:28.556 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(736), op: ExtentFlush { repair_id: ReconciliationId(736), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.556 INFO [1] client ExtentFlush { repair_id: ReconciliationId(736), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.556 INFO [2] received reconcile message
Sep 22 23:22:28.556 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(736), op: ExtentFlush { repair_id: ReconciliationId(736), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.556 INFO [2] client ExtentFlush { repair_id: ReconciliationId(736), extent_id: 57, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.556 DEBG 736 Flush extent 57 with f:2 g:2
Sep 22 23:22:28.556 DEBG Flush just extent 57 with f:2 and g:2
Sep 22 23:22:28.556 DEBG [1] It's time to notify for 736
Sep 22 23:22:28.556 INFO Completion from [1] id:736 status:true
Sep 22 23:22:28.556 INFO [737/752] Repair commands completed
Sep 22 23:22:28.556 INFO Pop front: ReconcileIO { id: ReconciliationId(737), op: ExtentClose { repair_id: ReconciliationId(737), extent_id: 57 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.556 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.556 INFO [0] received reconcile message
Sep 22 23:22:28.556 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(737), op: ExtentClose { repair_id: ReconciliationId(737), extent_id: 57 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.556 INFO [0] client ExtentClose { repair_id: ReconciliationId(737), extent_id: 57 }
Sep 22 23:22:28.556 INFO [1] received reconcile message
Sep 22 23:22:28.556 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(737), op: ExtentClose { repair_id: ReconciliationId(737), extent_id: 57 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.557 INFO [1] client ExtentClose { repair_id: ReconciliationId(737), extent_id: 57 }
Sep 22 23:22:28.557 INFO [2] received reconcile message
Sep 22 23:22:28.557 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(737), op: ExtentClose { repair_id: ReconciliationId(737), extent_id: 57 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.557 INFO [2] client ExtentClose { repair_id: ReconciliationId(737), extent_id: 57 }
Sep 22 23:22:28.557 DEBG 737 Close extent 57
Sep 22 23:22:28.557 DEBG 737 Close extent 57
Sep 22 23:22:28.557 DEBG 737 Close extent 57
Sep 22 23:22:28.558 DEBG [2] It's time to notify for 737
Sep 22 23:22:28.558 INFO Completion from [2] id:737 status:true
Sep 22 23:22:28.558 INFO [738/752] Repair commands completed
Sep 22 23:22:28.558 INFO Pop front: ReconcileIO { id: ReconciliationId(738), op: ExtentRepair { repair_id: ReconciliationId(738), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.558 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.558 INFO [0] received reconcile message
Sep 22 23:22:28.558 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(738), op: ExtentRepair { repair_id: ReconciliationId(738), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.558 INFO [0] client ExtentRepair { repair_id: ReconciliationId(738), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.558 INFO [0] Sending repair request ReconciliationId(738)
Sep 22 23:22:28.558 INFO [1] received reconcile message
Sep 22 23:22:28.558 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(738), op: ExtentRepair { repair_id: ReconciliationId(738), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.558 INFO [1] client ExtentRepair { repair_id: ReconciliationId(738), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.558 INFO [1] No action required ReconciliationId(738)
Sep 22 23:22:28.558 INFO [2] received reconcile message
Sep 22 23:22:28.558 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(738), op: ExtentRepair { repair_id: ReconciliationId(738), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.558 INFO [2] client ExtentRepair { repair_id: ReconciliationId(738), extent_id: 57, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.558 INFO [2] No action required ReconciliationId(738)
Sep 22 23:22:28.558 DEBG 738 Repair extent 57 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.558 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/039.copy"
Sep 22 23:22:28.621 INFO accepted connection, remote_addr: 127.0.0.1:37146, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.622 TRCE incoming request, uri: /extent/57/files, method: GET, req_id: 456a0d50-d134-4fc8-b9e9-ee8e794a4c55, remote_addr: 127.0.0.1:37146, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.622 INFO request completed, latency_us: 264, response_code: 200, uri: /extent/57/files, method: GET, req_id: 456a0d50-d134-4fc8-b9e9-ee8e794a4c55, remote_addr: 127.0.0.1:37146, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.622 INFO eid:57 Found repair files: ["039", "039.db"]
Sep 22 23:22:28.623 TRCE incoming request, uri: /newextent/57/data, method: GET, req_id: 721154bf-650c-48f8-931a-17a52b9a4dd1, remote_addr: 127.0.0.1:37146, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.623 INFO request completed, latency_us: 341, response_code: 200, uri: /newextent/57/data, method: GET, req_id: 721154bf-650c-48f8-931a-17a52b9a4dd1, remote_addr: 127.0.0.1:37146, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.628 TRCE incoming request, uri: /newextent/57/db, method: GET, req_id: 09fc977b-0d3b-4790-abf8-83c03db218b6, remote_addr: 127.0.0.1:37146, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.628 INFO request completed, latency_us: 302, response_code: 200, uri: /newextent/57/db, method: GET, req_id: 09fc977b-0d3b-4790-abf8-83c03db218b6, remote_addr: 127.0.0.1:37146, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.629 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/039.copy" to "/tmp/downstairs-zrMnlo6G/00/000/039.replace"
Sep 22 23:22:28.629 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.631 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/039.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.631 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/039"
Sep 22 23:22:28.631 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/039.db"
Sep 22 23:22:28.631 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.631 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/039.replace" to "/tmp/downstairs-zrMnlo6G/00/000/039.completed"
Sep 22 23:22:28.631 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.631 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.631 DEBG [0] It's time to notify for 738
Sep 22 23:22:28.631 INFO Completion from [0] id:738 status:true
Sep 22 23:22:28.631 INFO [739/752] Repair commands completed
Sep 22 23:22:28.631 INFO Pop front: ReconcileIO { id: ReconciliationId(739), op: ExtentReopen { repair_id: ReconciliationId(739), extent_id: 57 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.631 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.632 INFO [0] received reconcile message
Sep 22 23:22:28.632 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(739), op: ExtentReopen { repair_id: ReconciliationId(739), extent_id: 57 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.632 INFO [0] client ExtentReopen { repair_id: ReconciliationId(739), extent_id: 57 }
Sep 22 23:22:28.632 INFO [1] received reconcile message
Sep 22 23:22:28.632 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(739), op: ExtentReopen { repair_id: ReconciliationId(739), extent_id: 57 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.632 INFO [1] client ExtentReopen { repair_id: ReconciliationId(739), extent_id: 57 }
Sep 22 23:22:28.632 INFO [2] received reconcile message
Sep 22 23:22:28.632 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(739), op: ExtentReopen { repair_id: ReconciliationId(739), extent_id: 57 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.632 INFO [2] client ExtentReopen { repair_id: ReconciliationId(739), extent_id: 57 }
Sep 22 23:22:28.632 DEBG 739 Reopen extent 57
Sep 22 23:22:28.633 DEBG 739 Reopen extent 57
Sep 22 23:22:28.633 DEBG 739 Reopen extent 57
Sep 22 23:22:28.634 DEBG [2] It's time to notify for 739
Sep 22 23:22:28.634 INFO Completion from [2] id:739 status:true
Sep 22 23:22:28.634 INFO [740/752] Repair commands completed
Sep 22 23:22:28.634 INFO Pop front: ReconcileIO { id: ReconciliationId(740), op: ExtentFlush { repair_id: ReconciliationId(740), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.634 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.634 INFO [0] received reconcile message
Sep 22 23:22:28.634 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(740), op: ExtentFlush { repair_id: ReconciliationId(740), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.634 INFO [0] client ExtentFlush { repair_id: ReconciliationId(740), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.634 INFO [1] received reconcile message
Sep 22 23:22:28.634 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(740), op: ExtentFlush { repair_id: ReconciliationId(740), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.634 INFO [1] client ExtentFlush { repair_id: ReconciliationId(740), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.634 INFO [2] received reconcile message
Sep 22 23:22:28.634 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(740), op: ExtentFlush { repair_id: ReconciliationId(740), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.634 INFO [2] client ExtentFlush { repair_id: ReconciliationId(740), extent_id: 14, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.634 DEBG 740 Flush extent 14 with f:2 g:2
Sep 22 23:22:28.634 DEBG Flush just extent 14 with f:2 and g:2
Sep 22 23:22:28.634 DEBG [1] It's time to notify for 740
Sep 22 23:22:28.634 INFO Completion from [1] id:740 status:true
Sep 22 23:22:28.634 INFO [741/752] Repair commands completed
Sep 22 23:22:28.634 INFO Pop front: ReconcileIO { id: ReconciliationId(741), op: ExtentClose { repair_id: ReconciliationId(741), extent_id: 14 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.635 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.635 INFO [0] received reconcile message
Sep 22 23:22:28.635 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(741), op: ExtentClose { repair_id: ReconciliationId(741), extent_id: 14 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.635 INFO [0] client ExtentClose { repair_id: ReconciliationId(741), extent_id: 14 }
Sep 22 23:22:28.635 INFO [1] received reconcile message
Sep 22 23:22:28.635 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(741), op: ExtentClose { repair_id: ReconciliationId(741), extent_id: 14 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.635 INFO [1] client ExtentClose { repair_id: ReconciliationId(741), extent_id: 14 }
Sep 22 23:22:28.635 INFO [2] received reconcile message
Sep 22 23:22:28.635 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(741), op: ExtentClose { repair_id: ReconciliationId(741), extent_id: 14 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.635 INFO [2] client ExtentClose { repair_id: ReconciliationId(741), extent_id: 14 }
Sep 22 23:22:28.635 DEBG 741 Close extent 14
Sep 22 23:22:28.635 DEBG 741 Close extent 14
Sep 22 23:22:28.635 DEBG 741 Close extent 14
Sep 22 23:22:28.636 DEBG [2] It's time to notify for 741
Sep 22 23:22:28.636 INFO Completion from [2] id:741 status:true
Sep 22 23:22:28.636 INFO [742/752] Repair commands completed
Sep 22 23:22:28.636 INFO Pop front: ReconcileIO { id: ReconciliationId(742), op: ExtentRepair { repair_id: ReconciliationId(742), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.636 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.636 INFO [0] received reconcile message
Sep 22 23:22:28.636 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(742), op: ExtentRepair { repair_id: ReconciliationId(742), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.636 INFO [0] client ExtentRepair { repair_id: ReconciliationId(742), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.636 INFO [0] Sending repair request ReconciliationId(742)
Sep 22 23:22:28.636 INFO [1] received reconcile message
Sep 22 23:22:28.636 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(742), op: ExtentRepair { repair_id: ReconciliationId(742), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.636 INFO [1] client ExtentRepair { repair_id: ReconciliationId(742), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.636 INFO [1] No action required ReconciliationId(742)
Sep 22 23:22:28.636 INFO [2] received reconcile message
Sep 22 23:22:28.636 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(742), op: ExtentRepair { repair_id: ReconciliationId(742), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.636 INFO [2] client ExtentRepair { repair_id: ReconciliationId(742), extent_id: 14, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.636 INFO [2] No action required ReconciliationId(742)
Sep 22 23:22:28.636 DEBG 742 Repair extent 14 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.636 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/00E.copy"
Sep 22 23:22:28.657 ERRO [2] job id 1094 saw error GenericError("test error")
Sep 22 23:22:28.657 DEBG up_ds_listen was notified
Sep 22 23:22:28.657 DEBG up_ds_listen process 1094
Sep 22 23:22:28.657 DEBG [A] ack job 1094:95, : downstairs
Sep 22 23:22:28.657 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:28.659 DEBG Flush     :1094 extent_limit None deps:[JobId(1093), JobId(1092)] res:true f:36 g:1
Sep 22 23:22:28.659 INFO [lossy] sleeping 1 second
Sep 22 23:22:28.659 WARN returning error on read!
Sep 22 23:22:28.659 DEBG Read      :1095 deps:[JobId(1094)] res:false
Sep 22 23:22:28.665 DEBG Read      :1095 deps:[JobId(1094)] res:true
Sep 22 23:22:28.703 INFO accepted connection, remote_addr: 127.0.0.1:56670, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.703 TRCE incoming request, uri: /extent/14/files, method: GET, req_id: 2bed8d8a-06b4-490b-be53-7bf3beafe516, remote_addr: 127.0.0.1:56670, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.703 INFO request completed, latency_us: 264, response_code: 200, uri: /extent/14/files, method: GET, req_id: 2bed8d8a-06b4-490b-be53-7bf3beafe516, remote_addr: 127.0.0.1:56670, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.703 INFO eid:14 Found repair files: ["00E", "00E.db"]
Sep 22 23:22:28.704 TRCE incoming request, uri: /newextent/14/data, method: GET, req_id: 91137b77-534b-49a7-b8db-12d767dae7bd, remote_addr: 127.0.0.1:56670, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.704 INFO request completed, latency_us: 362, response_code: 200, uri: /newextent/14/data, method: GET, req_id: 91137b77-534b-49a7-b8db-12d767dae7bd, remote_addr: 127.0.0.1:56670, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.709 TRCE incoming request, uri: /newextent/14/db, method: GET, req_id: 060add22-2817-41a6-b7ae-98bd0e88b945, remote_addr: 127.0.0.1:56670, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.709 INFO request completed, latency_us: 301, response_code: 200, uri: /newextent/14/db, method: GET, req_id: 060add22-2817-41a6-b7ae-98bd0e88b945, remote_addr: 127.0.0.1:56670, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.710 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/00E.copy" to "/tmp/downstairs-zrMnlo6G/00/000/00E.replace"
Sep 22 23:22:28.710 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.712 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/00E.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.712 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00E"
Sep 22 23:22:28.712 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/00E.db"
Sep 22 23:22:28.712 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.712 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/00E.replace" to "/tmp/downstairs-zrMnlo6G/00/000/00E.completed"
Sep 22 23:22:28.712 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.712 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.712 DEBG [0] It's time to notify for 742
Sep 22 23:22:28.712 INFO Completion from [0] id:742 status:true
Sep 22 23:22:28.712 INFO [743/752] Repair commands completed
Sep 22 23:22:28.712 INFO Pop front: ReconcileIO { id: ReconciliationId(743), op: ExtentReopen { repair_id: ReconciliationId(743), extent_id: 14 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.712 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.712 INFO [0] received reconcile message
Sep 22 23:22:28.713 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(743), op: ExtentReopen { repair_id: ReconciliationId(743), extent_id: 14 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.713 INFO [0] client ExtentReopen { repair_id: ReconciliationId(743), extent_id: 14 }
Sep 22 23:22:28.713 INFO [1] received reconcile message
Sep 22 23:22:28.713 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(743), op: ExtentReopen { repair_id: ReconciliationId(743), extent_id: 14 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.713 INFO [1] client ExtentReopen { repair_id: ReconciliationId(743), extent_id: 14 }
Sep 22 23:22:28.713 INFO [2] received reconcile message
Sep 22 23:22:28.713 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(743), op: ExtentReopen { repair_id: ReconciliationId(743), extent_id: 14 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.713 INFO [2] client ExtentReopen { repair_id: ReconciliationId(743), extent_id: 14 }
Sep 22 23:22:28.713 DEBG 743 Reopen extent 14
Sep 22 23:22:28.714 DEBG 743 Reopen extent 14
Sep 22 23:22:28.714 DEBG 743 Reopen extent 14
Sep 22 23:22:28.715 DEBG [2] It's time to notify for 743
Sep 22 23:22:28.715 INFO Completion from [2] id:743 status:true
Sep 22 23:22:28.715 INFO [744/752] Repair commands completed
Sep 22 23:22:28.715 INFO Pop front: ReconcileIO { id: ReconciliationId(744), op: ExtentFlush { repair_id: ReconciliationId(744), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.715 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.715 INFO [0] received reconcile message
Sep 22 23:22:28.715 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(744), op: ExtentFlush { repair_id: ReconciliationId(744), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.715 INFO [0] client ExtentFlush { repair_id: ReconciliationId(744), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.715 INFO [1] received reconcile message
Sep 22 23:22:28.715 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(744), op: ExtentFlush { repair_id: ReconciliationId(744), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.715 INFO [1] client ExtentFlush { repair_id: ReconciliationId(744), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.715 INFO [2] received reconcile message
Sep 22 23:22:28.715 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(744), op: ExtentFlush { repair_id: ReconciliationId(744), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.715 INFO [2] client ExtentFlush { repair_id: ReconciliationId(744), extent_id: 8, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.715 DEBG 744 Flush extent 8 with f:2 g:2
Sep 22 23:22:28.715 DEBG Flush just extent 8 with f:2 and g:2
Sep 22 23:22:28.715 DEBG [1] It's time to notify for 744
Sep 22 23:22:28.715 INFO Completion from [1] id:744 status:true
Sep 22 23:22:28.715 INFO [745/752] Repair commands completed
Sep 22 23:22:28.715 INFO Pop front: ReconcileIO { id: ReconciliationId(745), op: ExtentClose { repair_id: ReconciliationId(745), extent_id: 8 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.715 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.716 INFO [0] received reconcile message
Sep 22 23:22:28.716 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(745), op: ExtentClose { repair_id: ReconciliationId(745), extent_id: 8 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.716 INFO [0] client ExtentClose { repair_id: ReconciliationId(745), extent_id: 8 }
Sep 22 23:22:28.716 INFO [1] received reconcile message
Sep 22 23:22:28.716 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(745), op: ExtentClose { repair_id: ReconciliationId(745), extent_id: 8 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.716 INFO [1] client ExtentClose { repair_id: ReconciliationId(745), extent_id: 8 }
Sep 22 23:22:28.716 INFO [2] received reconcile message
Sep 22 23:22:28.716 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(745), op: ExtentClose { repair_id: ReconciliationId(745), extent_id: 8 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.716 INFO [2] client ExtentClose { repair_id: ReconciliationId(745), extent_id: 8 }
Sep 22 23:22:28.716 DEBG 745 Close extent 8
Sep 22 23:22:28.716 DEBG 745 Close extent 8
Sep 22 23:22:28.716 DEBG 745 Close extent 8
Sep 22 23:22:28.717 DEBG [2] It's time to notify for 745
Sep 22 23:22:28.717 INFO Completion from [2] id:745 status:true
Sep 22 23:22:28.717 INFO [746/752] Repair commands completed
Sep 22 23:22:28.717 INFO Pop front: ReconcileIO { id: ReconciliationId(746), op: ExtentRepair { repair_id: ReconciliationId(746), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.717 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.717 INFO [0] received reconcile message
Sep 22 23:22:28.717 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(746), op: ExtentRepair { repair_id: ReconciliationId(746), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.717 INFO [0] client ExtentRepair { repair_id: ReconciliationId(746), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.717 INFO [0] Sending repair request ReconciliationId(746)
Sep 22 23:22:28.717 INFO [1] received reconcile message
Sep 22 23:22:28.717 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(746), op: ExtentRepair { repair_id: ReconciliationId(746), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.717 INFO [1] client ExtentRepair { repair_id: ReconciliationId(746), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.717 INFO [1] No action required ReconciliationId(746)
Sep 22 23:22:28.717 INFO [2] received reconcile message
Sep 22 23:22:28.717 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(746), op: ExtentRepair { repair_id: ReconciliationId(746), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.717 INFO [2] client ExtentRepair { repair_id: ReconciliationId(746), extent_id: 8, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.717 INFO [2] No action required ReconciliationId(746)
Sep 22 23:22:28.717 DEBG 746 Repair extent 8 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.717 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/008.copy"
Sep 22 23:22:28.735 ERRO [0] job id 1095 saw error GenericError("test error")
Sep 22 23:22:28.782 INFO accepted connection, remote_addr: 127.0.0.1:39346, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.782 TRCE incoming request, uri: /extent/8/files, method: GET, req_id: 53b51468-eb9d-40ab-b69e-e41875551f61, remote_addr: 127.0.0.1:39346, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.783 INFO request completed, latency_us: 233, response_code: 200, uri: /extent/8/files, method: GET, req_id: 53b51468-eb9d-40ab-b69e-e41875551f61, remote_addr: 127.0.0.1:39346, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.783 INFO eid:8 Found repair files: ["008", "008.db"]
Sep 22 23:22:28.783 TRCE incoming request, uri: /newextent/8/data, method: GET, req_id: 74dbf164-1dde-492a-80ee-f65c5bf763b3, remote_addr: 127.0.0.1:39346, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.784 INFO request completed, latency_us: 333, response_code: 200, uri: /newextent/8/data, method: GET, req_id: 74dbf164-1dde-492a-80ee-f65c5bf763b3, remote_addr: 127.0.0.1:39346, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.788 TRCE incoming request, uri: /newextent/8/db, method: GET, req_id: cd545e92-1a51-4dd5-8fcf-30e62d6ebb67, remote_addr: 127.0.0.1:39346, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.789 INFO request completed, latency_us: 290, response_code: 200, uri: /newextent/8/db, method: GET, req_id: cd545e92-1a51-4dd5-8fcf-30e62d6ebb67, remote_addr: 127.0.0.1:39346, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.790 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/008.copy" to "/tmp/downstairs-zrMnlo6G/00/000/008.replace"
Sep 22 23:22:28.790 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.791 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/008.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.791 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/008"
Sep 22 23:22:28.791 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/008.db"
Sep 22 23:22:28.791 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.791 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/008.replace" to "/tmp/downstairs-zrMnlo6G/00/000/008.completed"
Sep 22 23:22:28.791 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.791 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.791 DEBG [0] It's time to notify for 746
Sep 22 23:22:28.792 INFO Completion from [0] id:746 status:true
Sep 22 23:22:28.792 INFO [747/752] Repair commands completed
Sep 22 23:22:28.792 INFO Pop front: ReconcileIO { id: ReconciliationId(747), op: ExtentReopen { repair_id: ReconciliationId(747), extent_id: 8 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.792 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.792 INFO [0] received reconcile message
Sep 22 23:22:28.792 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(747), op: ExtentReopen { repair_id: ReconciliationId(747), extent_id: 8 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.792 INFO [0] client ExtentReopen { repair_id: ReconciliationId(747), extent_id: 8 }
Sep 22 23:22:28.792 INFO [1] received reconcile message
Sep 22 23:22:28.792 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(747), op: ExtentReopen { repair_id: ReconciliationId(747), extent_id: 8 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.792 INFO [1] client ExtentReopen { repair_id: ReconciliationId(747), extent_id: 8 }
Sep 22 23:22:28.792 INFO [2] received reconcile message
Sep 22 23:22:28.792 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(747), op: ExtentReopen { repair_id: ReconciliationId(747), extent_id: 8 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.792 INFO [2] client ExtentReopen { repair_id: ReconciliationId(747), extent_id: 8 }
Sep 22 23:22:28.792 DEBG 747 Reopen extent 8
Sep 22 23:22:28.793 DEBG 747 Reopen extent 8
Sep 22 23:22:28.793 DEBG 747 Reopen extent 8
Sep 22 23:22:28.794 DEBG [2] It's time to notify for 747
Sep 22 23:22:28.794 INFO Completion from [2] id:747 status:true
Sep 22 23:22:28.794 INFO [748/752] Repair commands completed
Sep 22 23:22:28.794 INFO Pop front: ReconcileIO { id: ReconciliationId(748), op: ExtentFlush { repair_id: ReconciliationId(748), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.794 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.794 INFO [0] received reconcile message
Sep 22 23:22:28.794 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(748), op: ExtentFlush { repair_id: ReconciliationId(748), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.794 INFO [0] client ExtentFlush { repair_id: ReconciliationId(748), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.794 INFO [1] received reconcile message
Sep 22 23:22:28.794 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(748), op: ExtentFlush { repair_id: ReconciliationId(748), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, New]) }, : downstairs
Sep 22 23:22:28.794 INFO [1] client ExtentFlush { repair_id: ReconciliationId(748), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.794 INFO [2] received reconcile message
Sep 22 23:22:28.794 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(748), op: ExtentFlush { repair_id: ReconciliationId(748), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }, state: ClientData([Done, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.794 INFO [2] client ExtentFlush { repair_id: ReconciliationId(748), extent_id: 114, client_id: ClientId(1), flush_number: 2, gen_number: 2 }
Sep 22 23:22:28.794 DEBG 748 Flush extent 114 with f:2 g:2
Sep 22 23:22:28.794 DEBG Flush just extent 114 with f:2 and g:2
Sep 22 23:22:28.794 DEBG [1] It's time to notify for 748
Sep 22 23:22:28.794 INFO Completion from [1] id:748 status:true
Sep 22 23:22:28.795 INFO [749/752] Repair commands completed
Sep 22 23:22:28.795 INFO Pop front: ReconcileIO { id: ReconciliationId(749), op: ExtentClose { repair_id: ReconciliationId(749), extent_id: 114 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.795 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.795 INFO [0] received reconcile message
Sep 22 23:22:28.795 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(749), op: ExtentClose { repair_id: ReconciliationId(749), extent_id: 114 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.795 INFO [0] client ExtentClose { repair_id: ReconciliationId(749), extent_id: 114 }
Sep 22 23:22:28.795 INFO [1] received reconcile message
Sep 22 23:22:28.795 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(749), op: ExtentClose { repair_id: ReconciliationId(749), extent_id: 114 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.795 INFO [1] client ExtentClose { repair_id: ReconciliationId(749), extent_id: 114 }
Sep 22 23:22:28.795 INFO [2] received reconcile message
Sep 22 23:22:28.795 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(749), op: ExtentClose { repair_id: ReconciliationId(749), extent_id: 114 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.795 INFO [2] client ExtentClose { repair_id: ReconciliationId(749), extent_id: 114 }
Sep 22 23:22:28.795 DEBG 749 Close extent 114
Sep 22 23:22:28.795 DEBG 749 Close extent 114
Sep 22 23:22:28.795 DEBG 749 Close extent 114
Sep 22 23:22:28.796 DEBG [2] It's time to notify for 749
Sep 22 23:22:28.796 INFO Completion from [2] id:749 status:true
Sep 22 23:22:28.796 INFO [750/752] Repair commands completed
Sep 22 23:22:28.796 INFO Pop front: ReconcileIO { id: ReconciliationId(750), op: ExtentRepair { repair_id: ReconciliationId(750), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.796 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.796 INFO [0] received reconcile message
Sep 22 23:22:28.796 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(750), op: ExtentRepair { repair_id: ReconciliationId(750), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.796 INFO [0] client ExtentRepair { repair_id: ReconciliationId(750), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.796 INFO [0] Sending repair request ReconciliationId(750)
Sep 22 23:22:28.796 INFO [1] received reconcile message
Sep 22 23:22:28.796 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(750), op: ExtentRepair { repair_id: ReconciliationId(750), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.796 INFO [1] client ExtentRepair { repair_id: ReconciliationId(750), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.796 INFO [1] No action required ReconciliationId(750)
Sep 22 23:22:28.796 INFO [2] received reconcile message
Sep 22 23:22:28.796 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(750), op: ExtentRepair { repair_id: ReconciliationId(750), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }, state: ClientData([InProgress, Done, InProgress]) }, : downstairs
Sep 22 23:22:28.796 INFO [2] client ExtentRepair { repair_id: ReconciliationId(750), extent_id: 114, source_client_id: ClientId(1), source_repair_address: 127.0.0.1:52864, dest_clients: [ClientId(0)] }
Sep 22 23:22:28.796 INFO [2] No action required ReconciliationId(750)
Sep 22 23:22:28.796 DEBG 750 Repair extent 114 source:[1] 127.0.0.1:52864 dest:[ClientId(0)]
Sep 22 23:22:28.796 INFO Created copy dir "/tmp/downstairs-zrMnlo6G/00/000/072.copy"
Sep 22 23:22:28.861 INFO accepted connection, remote_addr: 127.0.0.1:38532, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.861 TRCE incoming request, uri: /extent/114/files, method: GET, req_id: 23b65fae-a1f5-44ab-9b64-bd94423e2b78, remote_addr: 127.0.0.1:38532, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.862 INFO request completed, latency_us: 245, response_code: 200, uri: /extent/114/files, method: GET, req_id: 23b65fae-a1f5-44ab-9b64-bd94423e2b78, remote_addr: 127.0.0.1:38532, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.862 INFO eid:114 Found repair files: ["072", "072.db"]
Sep 22 23:22:28.862 TRCE incoming request, uri: /newextent/114/data, method: GET, req_id: 7406db28-6503-47d5-893f-a4fcda6dafd5, remote_addr: 127.0.0.1:38532, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.863 INFO request completed, latency_us: 334, response_code: 200, uri: /newextent/114/data, method: GET, req_id: 7406db28-6503-47d5-893f-a4fcda6dafd5, remote_addr: 127.0.0.1:38532, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.868 TRCE incoming request, uri: /newextent/114/db, method: GET, req_id: 9b1cb398-9752-4fbe-a136-d8b96bece8ff, remote_addr: 127.0.0.1:38532, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.868 INFO request completed, latency_us: 403, response_code: 200, uri: /newextent/114/db, method: GET, req_id: 9b1cb398-9752-4fbe-a136-d8b96bece8ff, remote_addr: 127.0.0.1:38532, local_addr: 127.0.0.1:52864, task: repair
Sep 22 23:22:28.869 INFO Repair files downloaded, move directory "/tmp/downstairs-zrMnlo6G/00/000/072.copy" to "/tmp/downstairs-zrMnlo6G/00/000/072.replace"
Sep 22 23:22:28.869 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.870 INFO Copy files from "/tmp/downstairs-zrMnlo6G/00/000/072.replace" in "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.871 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/072"
Sep 22 23:22:28.871 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000/072.db"
Sep 22 23:22:28.871 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.871 INFO Move directory  "/tmp/downstairs-zrMnlo6G/00/000/072.replace" to "/tmp/downstairs-zrMnlo6G/00/000/072.completed"
Sep 22 23:22:28.871 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.871 DEBG fsync completed for: "/tmp/downstairs-zrMnlo6G/00/000"
Sep 22 23:22:28.871 DEBG [0] It's time to notify for 750
Sep 22 23:22:28.871 INFO Completion from [0] id:750 status:true
Sep 22 23:22:28.871 INFO [751/752] Repair commands completed
Sep 22 23:22:28.871 INFO Pop front: ReconcileIO { id: ReconciliationId(751), op: ExtentReopen { repair_id: ReconciliationId(751), extent_id: 114 }, state: ClientData([New, New, New]) }
Sep 22 23:22:28.871 INFO Sent repair work, now wait for resp
Sep 22 23:22:28.871 INFO [0] received reconcile message
Sep 22 23:22:28.871 INFO [0] rep_in_progress: return ReconcileIO { id: ReconciliationId(751), op: ExtentReopen { repair_id: ReconciliationId(751), extent_id: 114 }, state: ClientData([InProgress, New, New]) }, : downstairs
Sep 22 23:22:28.871 INFO [0] client ExtentReopen { repair_id: ReconciliationId(751), extent_id: 114 }
Sep 22 23:22:28.871 INFO [1] received reconcile message
Sep 22 23:22:28.871 INFO [1] rep_in_progress: return ReconcileIO { id: ReconciliationId(751), op: ExtentReopen { repair_id: ReconciliationId(751), extent_id: 114 }, state: ClientData([InProgress, InProgress, New]) }, : downstairs
Sep 22 23:22:28.871 INFO [1] client ExtentReopen { repair_id: ReconciliationId(751), extent_id: 114 }
Sep 22 23:22:28.871 INFO [2] received reconcile message
Sep 22 23:22:28.871 INFO [2] rep_in_progress: return ReconcileIO { id: ReconciliationId(751), op: ExtentReopen { repair_id: ReconciliationId(751), extent_id: 114 }, state: ClientData([InProgress, InProgress, InProgress]) }, : downstairs
Sep 22 23:22:28.871 INFO [2] client ExtentReopen { repair_id: ReconciliationId(751), extent_id: 114 }
Sep 22 23:22:28.872 DEBG 751 Reopen extent 114
Sep 22 23:22:28.872 DEBG 751 Reopen extent 114
Sep 22 23:22:28.873 DEBG 751 Reopen extent 114
Sep 22 23:22:28.874 DEBG [2] It's time to notify for 751
Sep 22 23:22:28.874 INFO Completion from [2] id:751 status:true
Sep 22 23:22:28.874 INFO [752/752] Repair commands completed
Sep 22 23:22:28.874 INFO 188 extents repaired in 14.739 ave:0.0784
Sep 22 23:22:28.874 INFO All required repair work is completed
Sep 22 23:22:28.874 INFO Set Downstairs and Upstairs active after repairs
Sep 22 23:22:28.874 INFO 9144e02c-c312-47c4-9b1c-f03618834608 is now active with session: ec8dc28c-8140-4e2d-b538-f84f8abf147b
Sep 22 23:22:28.874 INFO Notify all downstairs, region set compare is done.
Sep 22 23:22:28.874 INFO Set check for repair
Sep 22 23:22:28.874 INFO [1] 127.0.0.1:48339 task reports connection:true
Sep 22 23:22:28.874 INFO 9144e02c-c312-47c4-9b1c-f03618834608 Active Active Active
Sep 22 23:22:28.874 INFO Set check for repair
Sep 22 23:22:28.874 INFO [2] 127.0.0.1:33021 task reports connection:true
Sep 22 23:22:28.874 INFO 9144e02c-c312-47c4-9b1c-f03618834608 Active Active Active
Sep 22 23:22:28.874 INFO Set check for repair
Sep 22 23:22:28.874 INFO [0] received reconcile message
Sep 22 23:22:28.874 INFO [0] All repairs completed, exit
Sep 22 23:22:28.874 INFO [0] Starts cmd_loop
Sep 22 23:22:28.874 INFO [1] received reconcile message
Sep 22 23:22:28.874 INFO [1] All repairs completed, exit
Sep 22 23:22:28.874 INFO [1] Starts cmd_loop
Sep 22 23:22:28.874 INFO [2] received reconcile message
Sep 22 23:22:28.874 INFO [2] All repairs completed, exit
Sep 22 23:22:28.874 INFO [2] Starts cmd_loop
The guest has finished waiting for activation
Sep 22 23:22:28.893 DEBG IO Read  1000 has deps []
Sep 22 23:22:29.069 DEBG [rc] retire 1094 clears [JobId(1093), JobId(1094)], : downstairs
Sep 22 23:22:29.069 DEBG IO Flush 1096 has deps [JobId(1095)]
Sep 22 23:22:29.073 DEBG Flush     :1096 extent_limit None deps:[JobId(1095)] res:true f:37 g:1
Sep 22 23:22:29.073 INFO [lossy] sleeping 1 second
Sep 22 23:22:29.451 DEBG [2] Read AckReady 1095, : downstairs
Sep 22 23:22:29.452 DEBG up_ds_listen was notified
Sep 22 23:22:29.452 DEBG up_ds_listen process 1095
Sep 22 23:22:29.452 DEBG [A] ack job 1095:96, : downstairs
Sep 22 23:22:29.505 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:29.508 INFO [lossy] skipping 1096
Sep 22 23:22:29.508 DEBG Flush     :1096 extent_limit None deps:[JobId(1095)] res:true f:37 g:1
Sep 22 23:22:29.508 INFO [lossy] sleeping 1 second
Sep 22 23:22:29.577 DEBG IO Read  1097 has deps [JobId(1096)]
Sep 22 23:22:29.977 DEBG up_ds_listen was notified
Sep 22 23:22:29.977 DEBG up_ds_listen process 1096
Sep 22 23:22:29.977 DEBG [A] ack job 1096:97, : downstairs
Sep 22 23:22:29.977 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:29.983 DEBG IO Flush 1098 has deps [JobId(1097), JobId(1096)]
Sep 22 23:22:30.011 DEBG Read      :1095 deps:[JobId(1094)] res:true
Sep 22 23:22:30.035 DEBG Flush     :1096 extent_limit None deps:[JobId(1095)] res:true f:37 g:1
Sep 22 23:22:30.035 INFO [lossy] skipping 1097
Sep 22 23:22:30.052 DEBG Read      :1097 deps:[JobId(1096)] res:true
Sep 22 23:22:30.633 DEBG [rc] retire 1096 clears [JobId(1095), JobId(1096)], : downstairs
Sep 22 23:22:30.633 WARN returning error on read!
Sep 22 23:22:30.633 DEBG Read      :1097 deps:[JobId(1096)] res:false
Sep 22 23:22:30.634 INFO [lossy] skipping 1097
Sep 22 23:22:30.641 DEBG Read      :1097 deps:[JobId(1096)] res:true
Sep 22 23:22:30.663 INFO [lossy] skipping 1097
Sep 22 23:22:30.686 DEBG Read      :1097 deps:[JobId(1096)] res:true
Sep 22 23:22:30.708 ERRO [2] job id 1097 saw error GenericError("test error")
Sep 22 23:22:30.710 DEBG Flush     :1098 extent_limit None deps:[JobId(1097), JobId(1096)] res:true f:38 g:1
Sep 22 23:22:30.711 INFO [lossy] sleeping 1 second
Sep 22 23:22:31.098 DEBG [1] Read AckReady 1097, : downstairs
Sep 22 23:22:31.099 DEBG up_ds_listen was notified
Sep 22 23:22:31.099 DEBG up_ds_listen process 1097
Sep 22 23:22:31.099 DEBG [A] ack job 1097:98, : downstairs
Sep 22 23:22:31.152 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:31.155 INFO [lossy] sleeping 1 second
Sep 22 23:22:31.157 WARN returning error on flush!
Sep 22 23:22:31.157 DEBG Flush     :1098 extent_limit None deps:[JobId(1097), JobId(1096)] res:false f:38 g:1
Sep 22 23:22:31.157 DEBG Flush     :1098 extent_limit None deps:[JobId(1097), JobId(1096)] res:true f:38 g:1
Sep 22 23:22:31.157 INFO [lossy] sleeping 1 second
Sep 22 23:22:31.486 DEBG Read      :1000 deps:[] res:true
Sep 22 23:22:31.499 DEBG IO Read  1099 has deps [JobId(1098)]
Sep 22 23:22:32.188 ERRO [2] job id 1098 saw error GenericError("test error")
Sep 22 23:22:32.194 DEBG up_ds_listen was notified
Sep 22 23:22:32.194 DEBG up_ds_listen process 1098
Sep 22 23:22:32.194 DEBG [A] ack job 1098:99, : downstairs
Sep 22 23:22:32.194 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:32.200 DEBG IO Flush 1100 has deps [JobId(1099), JobId(1098)]
Sep 22 23:22:32.201 INFO [lossy] sleeping 1 second
Sep 22 23:22:32.201 DEBG Flush     :1098 extent_limit None deps:[JobId(1097), JobId(1096)] res:true f:38 g:1
Sep 22 23:22:32.201 INFO [lossy] sleeping 1 second
Sep 22 23:22:32.201 INFO [lossy] sleeping 1 second
Sep 22 23:22:32.201 DEBG [rc] retire 1098 clears [JobId(1097), JobId(1098)], : downstairs
Sep 22 23:22:32.941 DEBG Read      :1000 deps:[] res:true
Sep 22 23:22:33.202 INFO [lossy] skipping 1099
Sep 22 23:22:33.224 DEBG Read      :1099 deps:[JobId(1098)] res:true
Sep 22 23:22:33.246 WARN returning error on read!
Sep 22 23:22:33.246 DEBG Read      :1099 deps:[JobId(1098)] res:false
Sep 22 23:22:33.274 DEBG Read      :1099 deps:[JobId(1098)] res:true
Sep 22 23:22:33.326 DEBG Read      :1099 deps:[JobId(1098)] res:true
Sep 22 23:22:33.348 ERRO [0] job id 1099 saw error GenericError("test error")
Sep 22 23:22:33.351 DEBG Flush     :1100 extent_limit None deps:[JobId(1099), JobId(1098)] res:true f:39 g:1
Sep 22 23:22:33.351 INFO [lossy] sleeping 1 second
Sep 22 23:22:33.352 DEBG Flush     :1100 extent_limit None deps:[JobId(1099), JobId(1098)] res:true f:39 g:1
Sep 22 23:22:33.352 INFO [lossy] sleeping 1 second
Sep 22 23:22:33.353 DEBG Flush     :1100 extent_limit None deps:[JobId(1099), JobId(1098)] res:true f:39 g:1
Sep 22 23:22:33.353 INFO [lossy] sleeping 1 second
Sep 22 23:22:33.849 DEBG [0] Read AckReady 1099, : downstairs
Sep 22 23:22:34.179 DEBG [1] Read already AckReady 1099, : downstairs
Sep 22 23:22:34.352 DEBG Read      :1000 deps:[] res:true
Sep 22 23:22:34.509 DEBG [2] Read already AckReady 1099, : downstairs
Sep 22 23:22:34.511 DEBG up_ds_listen was notified
Sep 22 23:22:34.511 DEBG up_ds_listen process 1099
Sep 22 23:22:34.511 DEBG [A] ack job 1099:100, : downstairs
Sep 22 23:22:34.564 DEBG up_ds_listen process 1100
Sep 22 23:22:34.564 DEBG [A] ack job 1100:101, : downstairs
Sep 22 23:22:34.564 DEBG [rc] retire 1100 clears [JobId(1099), JobId(1100)], : downstairs
Sep 22 23:22:34.564 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:22:34.564 DEBG up_ds_listen was notified
Sep 22 23:22:34.564 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:22:34.566 INFO [lossy] sleeping 1 second
Sep 22 23:22:34.566 INFO [lossy] sleeping 1 second
Sep 22 23:22:34.566 INFO [lossy] sleeping 1 second
Sep 22 23:22:34.566 DEBG IO Read  1101 has deps []
Sep 22 23:22:35.066 DEBG IO Flush 1102 has deps [JobId(1101)]
Sep 22 23:22:35.572 DEBG Read      :1101 deps:[] res:true
Sep 22 23:22:35.574 INFO Checking if live repair is needed
Sep 22 23:22:35.574 INFO No Live Repair required at this time
Sep 22 23:22:35.574 DEBG IO Flush 1001 has deps [JobId(1000)]
Sep 22 23:22:35.600 DEBG Read      :1101 deps:[] res:true
Sep 22 23:22:35.628 DEBG Read      :1101 deps:[] res:true
Sep 22 23:22:35.653 DEBG Flush     :1102 extent_limit None deps:[JobId(1101)] res:true f:40 g:1
Sep 22 23:22:35.653 INFO [lossy] sleeping 1 second
Sep 22 23:22:35.654 DEBG Flush     :1102 extent_limit None deps:[JobId(1101)] res:true f:40 g:1
Sep 22 23:22:35.654 INFO [lossy] sleeping 1 second
Sep 22 23:22:35.655 INFO [lossy] skipping 1102
Sep 22 23:22:35.655 DEBG Flush     :1102 extent_limit None deps:[JobId(1101)] res:true f:40 g:1
Sep 22 23:22:35.655 INFO [lossy] sleeping 1 second
Sep 22 23:22:35.749 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:22:35.844 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:22:35.949 DEBG Flush     :1001 extent_limit None deps:[JobId(1000)] res:true f:2 g:2
Sep 22 23:22:36.127 DEBG [0] Read AckReady 1101, : downstairs
Sep 22 23:22:36.457 DEBG [1] Read already AckReady 1101, : downstairs
Sep 22 23:22:36.789 DEBG [2] Read already AckReady 1101, : downstairs
Sep 22 23:22:36.791 DEBG up_ds_listen was notified
Sep 22 23:22:36.791 DEBG up_ds_listen process 1101
Sep 22 23:22:36.792 DEBG [A] ack job 1101:102, : downstairs
Sep 22 23:22:36.844 DEBG up_ds_listen process 1102
Sep 22 23:22:36.844 DEBG [A] ack job 1102:103, : downstairs
Sep 22 23:22:36.844 DEBG [rc] retire 1102 clears [JobId(1101), JobId(1102)], : downstairs
Sep 22 23:22:36.844 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:22:36.844 DEBG up_ds_listen was notified
Sep 22 23:22:36.844 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:22:36.846 INFO [lossy] sleeping 1 second
Sep 22 23:22:36.846 INFO [lossy] sleeping 1 second
Sep 22 23:22:36.846 INFO [lossy] sleeping 1 second
Sep 22 23:22:36.846 DEBG IO Read  1103 has deps []
Sep 22 23:22:37.347 DEBG IO Flush 1104 has deps [JobId(1103)]
test test::integration_test_problematic_downstairs has been running for over 60 seconds
Sep 22 23:22:37.847 INFO [lossy] skipping 1103
Sep 22 23:22:37.847 INFO [lossy] skipping 1104
Sep 22 23:22:37.847 INFO [lossy] skipping 1103
Sep 22 23:22:37.853 DEBG Read      :1103 deps:[] res:true
Sep 22 23:22:37.880 DEBG Read      :1103 deps:[] res:true
Sep 22 23:22:37.902 INFO [lossy] skipping 1103
Sep 22 23:22:37.908 DEBG Read      :1103 deps:[] res:true
Sep 22 23:22:37.932 INFO [lossy] skipping 1104
Sep 22 23:22:37.932 INFO [lossy] skipping 1104
Sep 22 23:22:37.932 DEBG Flush     :1104 extent_limit None deps:[JobId(1103)] res:true f:41 g:1
Sep 22 23:22:37.933 INFO [lossy] skipping 1104
Sep 22 23:22:37.933 DEBG Flush     :1104 extent_limit None deps:[JobId(1103)] res:true f:41 g:1
Sep 22 23:22:37.933 INFO [lossy] sleeping 1 second
Sep 22 23:22:37.934 WARN returning error on flush!
Sep 22 23:22:37.934 DEBG Flush     :1104 extent_limit None deps:[JobId(1103)] res:false f:41 g:1
Sep 22 23:22:37.935 INFO [lossy] skipping 1104
Sep 22 23:22:37.935 DEBG Flush     :1104 extent_limit None deps:[JobId(1103)] res:true f:41 g:1
Sep 22 23:22:37.935 INFO [lossy] sleeping 1 second
test test::integration_test_volume_replace_downstairs_then_takeover has been running for over 60 seconds
Sep 22 23:22:38.408 DEBG [0] Read AckReady 1103, : downstairs
Sep 22 23:22:38.738 DEBG [1] Read already AckReady 1103, : downstairs
Sep 22 23:22:39.068 DEBG [2] Read already AckReady 1103, : downstairs
Sep 22 23:22:39.074 ERRO [2] job id 1104 saw error GenericError("test error")
Sep 22 23:22:39.074 DEBG up_ds_listen was notified
Sep 22 23:22:39.075 DEBG up_ds_listen process 1103
Sep 22 23:22:39.075 DEBG [A] ack job 1103:104, : downstairs
Sep 22 23:22:39.131 DEBG up_ds_listen process 1104
Sep 22 23:22:39.131 DEBG [A] ack job 1104:105, : downstairs
Sep 22 23:22:39.131 DEBG [rc] retire 1104 clears [JobId(1103), JobId(1104)], : downstairs
Sep 22 23:22:39.131 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:22:39.131 DEBG up_ds_listen was notified
Sep 22 23:22:39.131 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:22:39.132 INFO [lossy] sleeping 1 second
Sep 22 23:22:39.132 INFO [lossy] sleeping 1 second
Sep 22 23:22:39.133 DEBG IO Read  1105 has deps []
Sep 22 23:22:39.152 DEBG Read      :1105 deps:[] res:true
Sep 22 23:22:39.557 DEBG [0] Read AckReady 1105, : downstairs
Sep 22 23:22:39.558 DEBG up_ds_listen was notified
Sep 22 23:22:39.558 DEBG up_ds_listen process 1105
Sep 22 23:22:39.558 DEBG [A] ack job 1105:106, : downstairs
Sep 22 23:22:39.611 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:39.612 DEBG IO Read  1106 has deps []
Sep 22 23:22:39.631 DEBG Read      :1106 deps:[] res:true
Sep 22 23:22:39.653 DEBG IO Flush 1107 has deps [JobId(1106), JobId(1105)]
Sep 22 23:22:39.655 DEBG Flush     :1107 extent_limit None deps:[JobId(1106), JobId(1105)] res:true f:42 g:1
Sep 22 23:22:40.035 DEBG [0] Read AckReady 1106, : downstairs
Sep 22 23:22:40.036 DEBG up_ds_listen was notified
Sep 22 23:22:40.036 DEBG up_ds_listen process 1106
Sep 22 23:22:40.036 DEBG [A] ack job 1106:107, : downstairs
Sep 22 23:22:40.089 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:40.090 DEBG IO Read  1108 has deps [JobId(1107)]
Sep 22 23:22:40.103 WARN returning error on read!
Sep 22 23:22:40.103 DEBG Read      :1108 deps:[JobId(1107)] res:false
Sep 22 23:22:40.103 INFO [lossy] skipping 1108
Sep 22 23:22:40.103 WARN returning error on read!
Sep 22 23:22:40.103 DEBG Read      :1108 deps:[JobId(1107)] res:false
Sep 22 23:22:40.103 INFO [lossy] skipping 1108
Sep 22 23:22:40.125 DEBG Read      :1108 deps:[JobId(1107)] res:true
Sep 22 23:22:40.183 DEBG Read      :1105 deps:[] res:true
Sep 22 23:22:40.240 DEBG Read      :1105 deps:[] res:true
Sep 22 23:22:40.262 ERRO [0] job id 1108 saw error GenericError("test error")
Sep 22 23:22:40.262 ERRO [0] job id 1108 saw error GenericError("test error")
Sep 22 23:22:40.262 DEBG IO Flush 1109 has deps [JobId(1108), JobId(1107)]
Sep 22 23:22:40.266 WARN returning error on flush!
Sep 22 23:22:40.266 DEBG Flush     :1109 extent_limit None deps:[JobId(1108), JobId(1107)] res:false f:43 g:1
Sep 22 23:22:40.266 INFO [lossy] skipping 1109
Sep 22 23:22:40.266 WARN returning error on flush!
Sep 22 23:22:40.266 DEBG Flush     :1109 extent_limit None deps:[JobId(1108), JobId(1107)] res:false f:43 g:1
Sep 22 23:22:40.266 DEBG Flush     :1109 extent_limit None deps:[JobId(1108), JobId(1107)] res:true f:43 g:1
Sep 22 23:22:40.345 DEBG Read      :1106 deps:[] res:true
Sep 22 23:22:40.375 DEBG Read      :1106 deps:[] res:true
Sep 22 23:22:40.745 DEBG [0] Read AckReady 1108, : downstairs
Sep 22 23:22:40.746 ERRO [0] job id 1109 saw error GenericError("test error")
Sep 22 23:22:40.746 ERRO [0] job id 1109 saw error GenericError("test error")
Sep 22 23:22:40.746 DEBG up_ds_listen was notified
Sep 22 23:22:40.746 DEBG up_ds_listen process 1108
Sep 22 23:22:40.746 DEBG [A] ack job 1108:109, : downstairs
Sep 22 23:22:40.799 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:41.583 DEBG IO Read  1110 has deps [JobId(1109)]
Sep 22 23:22:41.583 DEBG IO Flush 1111 has deps [JobId(1110), JobId(1109)]
Sep 22 23:22:41.595 INFO [lossy] sleeping 1 second
Sep 22 23:22:41.598 WARN returning error on flush!
Sep 22 23:22:41.598 DEBG Flush     :1107 extent_limit None deps:[JobId(1106), JobId(1105)] res:false f:42 g:1
Sep 22 23:22:41.598 DEBG Flush     :1107 extent_limit None deps:[JobId(1106), JobId(1105)] res:true f:42 g:1
Sep 22 23:22:41.598 INFO [lossy] skipping 1108
Sep 22 23:22:41.598 INFO [lossy] skipping 1110
Sep 22 23:22:41.598 INFO [lossy] skipping 1111
Sep 22 23:22:41.604 DEBG Read      :1108 deps:[JobId(1107)] res:true
Sep 22 23:22:41.626 WARN returning error on flush!
Sep 22 23:22:41.626 DEBG Flush     :1107 extent_limit None deps:[JobId(1106), JobId(1105)] res:false f:42 g:1
Sep 22 23:22:41.626 INFO [lossy] skipping 1107
Sep 22 23:22:41.626 DEBG Flush     :1107 extent_limit None deps:[JobId(1106), JobId(1105)] res:true f:42 g:1
Sep 22 23:22:41.626 INFO [lossy] skipping 1108
Sep 22 23:22:41.632 DEBG Read      :1108 deps:[JobId(1107)] res:true
Sep 22 23:22:42.115 ERRO [1] job id 1107 saw error GenericError("test error")
Sep 22 23:22:42.445 ERRO [2] job id 1107 saw error GenericError("test error")
Sep 22 23:22:42.446 DEBG up_ds_listen was notified
Sep 22 23:22:42.446 DEBG up_ds_listen process 1107
Sep 22 23:22:42.446 DEBG [A] ack job 1107:108, : downstairs
Sep 22 23:22:42.446 DEBG [rc] retire 1107 clears [JobId(1105), JobId(1106), JobId(1107)], : downstairs
Sep 22 23:22:42.446 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:42.449 INFO [lossy] skipping 1109
Sep 22 23:22:42.449 INFO [lossy] skipping 1110
Sep 22 23:22:42.449 DEBG Flush     :1109 extent_limit None deps:[JobId(1108), JobId(1107)] res:true f:43 g:1
Sep 22 23:22:42.449 INFO [lossy] skipping 1110
Sep 22 23:22:42.455 DEBG Read      :1110 deps:[JobId(1109)] res:true
Sep 22 23:22:42.477 DEBG Flush     :1109 extent_limit None deps:[JobId(1108), JobId(1107)] res:true f:43 g:1
Sep 22 23:22:42.482 DEBG Read      :1110 deps:[JobId(1109)] res:true
Sep 22 23:22:43.269 DEBG up_ds_listen was notified
Sep 22 23:22:43.269 DEBG up_ds_listen process 1109
Sep 22 23:22:43.269 DEBG [A] ack job 1109:110, : downstairs
Sep 22 23:22:43.269 DEBG [rc] retire 1109 clears [JobId(1108), JobId(1109)], : downstairs
Sep 22 23:22:43.269 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:43.290 DEBG Read      :1110 deps:[JobId(1109)] res:true
Sep 22 23:22:43.314 INFO [lossy] sleeping 1 second
Sep 22 23:22:43.316 DEBG Flush     :1111 extent_limit None deps:[JobId(1110), JobId(1109)] res:true f:44 g:1
Sep 22 23:22:43.316 INFO [lossy] sleeping 1 second
Sep 22 23:22:43.989 DEBG [1] Read AckReady 1110, : downstairs
Sep 22 23:22:44.330 DEBG [2] Read already AckReady 1110, : downstairs
Sep 22 23:22:44.332 DEBG up_ds_listen was notified
Sep 22 23:22:44.332 DEBG up_ds_listen process 1110
Sep 22 23:22:44.332 DEBG [A] ack job 1110:111, : downstairs
Sep 22 23:22:44.385 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:44.387 DEBG Flush     :1111 extent_limit None deps:[JobId(1110), JobId(1109)] res:true f:44 g:1
Sep 22 23:22:44.388 INFO [lossy] sleeping 1 second
Sep 22 23:22:44.388 DEBG Flush     :1111 extent_limit None deps:[JobId(1110), JobId(1109)] res:true f:44 g:1
Sep 22 23:22:44.388 INFO [lossy] sleeping 1 second
Sep 22 23:22:44.388 INFO [lossy] sleeping 1 second
Sep 22 23:22:44.435 DEBG IO Read  1112 has deps [JobId(1111)]
Sep 22 23:22:44.771 DEBG up_ds_listen was notified
Sep 22 23:22:44.771 DEBG up_ds_listen process 1111
Sep 22 23:22:44.771 DEBG [A] ack job 1111:112, : downstairs
Sep 22 23:22:44.771 DEBG [rc] retire 1111 clears [JobId(1110), JobId(1111)], : downstairs
Sep 22 23:22:44.771 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:44.889 DEBG IO Flush 1113 has deps [JobId(1112)]
Sep 22 23:22:45.395 DEBG Read      :1112 deps:[JobId(1111)] res:true
Sep 22 23:22:45.422 DEBG Read      :1112 deps:[JobId(1111)] res:true
Sep 22 23:22:45.444 INFO [lossy] skipping 1112
Sep 22 23:22:45.450 DEBG Read      :1112 deps:[JobId(1111)] res:true
Sep 22 23:22:45.474 DEBG Flush     :1113 extent_limit None deps:[JobId(1112)] res:true f:45 g:1
Sep 22 23:22:45.474 INFO [lossy] sleeping 1 second
Sep 22 23:22:45.522 DEBG Flush     :1113 extent_limit None deps:[JobId(1112)] res:true f:45 g:1
Sep 22 23:22:45.522 INFO [lossy] sleeping 1 second
Sep 22 23:22:45.523 DEBG Flush     :1113 extent_limit None deps:[JobId(1112)] res:true f:45 g:1
Sep 22 23:22:45.523 INFO [lossy] sleeping 1 second
Sep 22 23:22:45.851 DEBG [0] Read AckReady 1112, : downstairs
Sep 22 23:22:45.852 DEBG up_ds_listen was notified
Sep 22 23:22:45.852 DEBG up_ds_listen process 1112
Sep 22 23:22:45.852 DEBG [A] ack job 1112:113, : downstairs
Sep 22 23:22:45.905 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:46.662 DEBG up_ds_listen was notified
Sep 22 23:22:46.662 DEBG up_ds_listen process 1113
Sep 22 23:22:46.662 DEBG [A] ack job 1113:114, : downstairs
Sep 22 23:22:46.663 DEBG [rc] retire 1113 clears [JobId(1112), JobId(1113)], : downstairs
Sep 22 23:22:46.663 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:46.663 DEBG IO Read  1114 has deps []
Sep 22 23:22:46.663 DEBG IO Flush 1115 has deps [JobId(1114)]
Sep 22 23:22:46.663 INFO [lossy] sleeping 1 second
Sep 22 23:22:46.675 INFO [lossy] sleeping 1 second
Sep 22 23:22:46.681 DEBG Read      :1114 deps:[] res:true
Sep 22 23:22:46.704 DEBG Flush     :1115 extent_limit None deps:[JobId(1114)] res:true f:46 g:1
Sep 22 23:22:47.080 DEBG [1] Read AckReady 1114, : downstairs
Sep 22 23:22:47.081 DEBG up_ds_listen was notified
Sep 22 23:22:47.081 DEBG up_ds_listen process 1114
Sep 22 23:22:47.081 DEBG [A] ack job 1114:115, : downstairs
Sep 22 23:22:47.134 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:47.135 DEBG IO Read  1116 has deps [JobId(1115)]
Sep 22 23:22:47.153 DEBG Read      :1116 deps:[JobId(1115)] res:true
Sep 22 23:22:47.174 DEBG IO Flush 1117 has deps [JobId(1116), JobId(1115)]
Sep 22 23:22:47.176 DEBG Flush     :1117 extent_limit None deps:[JobId(1116), JobId(1115)] res:true f:47 g:1
Sep 22 23:22:47.552 DEBG [1] Read AckReady 1116, : downstairs
Sep 22 23:22:47.553 DEBG up_ds_listen was notified
Sep 22 23:22:47.553 DEBG up_ds_listen process 1116
Sep 22 23:22:47.553 DEBG [A] ack job 1116:117, : downstairs
Sep 22 23:22:47.606 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:47.607 DEBG IO Read  1118 has deps [JobId(1117)]
Sep 22 23:22:47.620 WARN returning error on read!
Sep 22 23:22:47.620 DEBG Read      :1118 deps:[JobId(1117)] res:false
Sep 22 23:22:47.620 INFO [lossy] skipping 1118
Sep 22 23:22:47.620 WARN returning error on read!
Sep 22 23:22:47.620 DEBG Read      :1118 deps:[JobId(1117)] res:false
Sep 22 23:22:47.620 INFO [lossy] skipping 1118
Sep 22 23:22:47.626 DEBG Read      :1118 deps:[JobId(1117)] res:true
Sep 22 23:22:47.647 ERRO [1] job id 1118 saw error GenericError("test error")
Sep 22 23:22:47.647 ERRO [1] job id 1118 saw error GenericError("test error")
Sep 22 23:22:48.024 DEBG [1] Read AckReady 1118, : downstairs
Sep 22 23:22:48.025 DEBG up_ds_listen was notified
Sep 22 23:22:48.025 DEBG up_ds_listen process 1118
Sep 22 23:22:48.026 DEBG [A] ack job 1118:119, : downstairs
Sep 22 23:22:48.078 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:48.079 WARN returning error on read!
Sep 22 23:22:48.079 DEBG Read      :1114 deps:[] res:false
Sep 22 23:22:48.080 INFO [lossy] skipping 1115
Sep 22 23:22:48.085 DEBG Read      :1114 deps:[] res:true
Sep 22 23:22:48.106 DEBG IO Flush 1119 has deps [JobId(1118), JobId(1117)]
Sep 22 23:22:48.107 INFO [lossy] skipping 1114
Sep 22 23:22:48.113 DEBG Read      :1114 deps:[] res:true
Sep 22 23:22:48.135 DEBG IO Read  1120 has deps [JobId(1119)]
Sep 22 23:22:48.135 ERRO [2] job id 1114 saw error GenericError("test error")
Sep 22 23:22:48.140 INFO [lossy] skipping 1119
Sep 22 23:22:48.140 DEBG Flush     :1119 extent_limit None deps:[JobId(1118), JobId(1117)] res:true f:48 g:1
Sep 22 23:22:48.147 INFO [lossy] sleeping 1 second
Sep 22 23:22:48.149 INFO [lossy] skipping 1115
Sep 22 23:22:48.149 INFO [lossy] skipping 1119
Sep 22 23:22:48.149 DEBG Flush     :1115 extent_limit None deps:[JobId(1114)] res:true f:46 g:1
Sep 22 23:22:48.149 INFO [lossy] skipping 1116
Sep 22 23:22:48.149 WARN 1117 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:48.155 DEBG Read      :1116 deps:[JobId(1115)] res:true
Sep 22 23:22:48.177 INFO [lossy] skipping 1115
Sep 22 23:22:48.177 DEBG Flush     :1115 extent_limit None deps:[JobId(1114)] res:true f:46 g:1
Sep 22 23:22:48.182 DEBG Read      :1116 deps:[JobId(1115)] res:true
Sep 22 23:22:48.968 DEBG up_ds_listen was notified
Sep 22 23:22:48.968 DEBG up_ds_listen process 1115
Sep 22 23:22:48.968 DEBG [A] ack job 1115:116, : downstairs
Sep 22 23:22:48.968 DEBG [rc] retire 1115 clears [JobId(1114), JobId(1115)], : downstairs
Sep 22 23:22:48.968 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:48.968 DEBG IO Flush 1121 has deps [JobId(1120), JobId(1119)]
Sep 22 23:22:48.971 DEBG Flush     :1117 extent_limit None deps:[JobId(1116), JobId(1115)] res:true f:47 g:1
Sep 22 23:22:48.971 INFO [lossy] skipping 1118
Sep 22 23:22:48.971 INFO [lossy] skipping 1119
Sep 22 23:22:48.971 INFO [lossy] skipping 1120
Sep 22 23:22:48.977 DEBG Read      :1118 deps:[JobId(1117)] res:true
Sep 22 23:22:48.999 WARN returning error on flush!
Sep 22 23:22:48.999 DEBG Flush     :1117 extent_limit None deps:[JobId(1116), JobId(1115)] res:false f:47 g:1
Sep 22 23:22:49.000 INFO [lossy] skipping 1117
Sep 22 23:22:49.000 DEBG Flush     :1117 extent_limit None deps:[JobId(1116), JobId(1115)] res:true f:47 g:1
Sep 22 23:22:49.000 WARN returning error on read!
Sep 22 23:22:49.000 DEBG Read      :1118 deps:[JobId(1117)] res:false
Sep 22 23:22:49.000 WARN 1119 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:49.006 DEBG Read      :1118 deps:[JobId(1117)] res:true
Sep 22 23:22:49.784 ERRO [2] job id 1117 saw error GenericError("test error")
Sep 22 23:22:49.784 ERRO [2] job id 1118 saw error GenericError("test error")
Sep 22 23:22:49.784 DEBG up_ds_listen was notified
Sep 22 23:22:49.784 DEBG up_ds_listen process 1117
Sep 22 23:22:49.784 DEBG [A] ack job 1117:118, : downstairs
Sep 22 23:22:49.784 DEBG [rc] retire 1117 clears [JobId(1116), JobId(1117)], : downstairs
Sep 22 23:22:49.784 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:49.790 DEBG Read      :1120 deps:[JobId(1119)] res:true
Sep 22 23:22:49.814 INFO [lossy] skipping 1119
Sep 22 23:22:49.814 INFO [lossy] skipping 1120
Sep 22 23:22:49.814 WARN returning error on flush!
Sep 22 23:22:49.815 DEBG Flush     :1119 extent_limit None deps:[JobId(1118), JobId(1117)] res:false f:48 g:1
Sep 22 23:22:49.815 DEBG Flush     :1119 extent_limit None deps:[JobId(1118), JobId(1117)] res:true f:48 g:1
Sep 22 23:22:49.815 WARN returning error on read!
Sep 22 23:22:49.815 DEBG Read      :1120 deps:[JobId(1119)] res:false
Sep 22 23:22:49.815 WARN 1121 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:22:49.815 WARN returning error on read!
Sep 22 23:22:49.815 DEBG Read      :1120 deps:[JobId(1119)] res:false
Sep 22 23:22:49.821 DEBG Read      :1120 deps:[JobId(1119)] res:true
Sep 22 23:22:49.842 WARN returning error on flush!
Sep 22 23:22:49.842 DEBG Flush     :1121 extent_limit None deps:[JobId(1120), JobId(1119)] res:false f:49 g:1
Sep 22 23:22:49.842 INFO [lossy] skipping 1121
Sep 22 23:22:49.842 INFO [lossy] skipping 1121
Sep 22 23:22:49.842 DEBG Flush     :1121 extent_limit None deps:[JobId(1120), JobId(1119)] res:true f:49 g:1
Sep 22 23:22:49.844 DEBG Flush     :1119 extent_limit None deps:[JobId(1118), JobId(1117)] res:true f:48 g:1
Sep 22 23:22:49.850 DEBG Read      :1120 deps:[JobId(1119)] res:true
Sep 22 23:22:50.351 ERRO [0] job id 1119 saw error GenericError("test error")
Sep 22 23:22:50.352 ERRO [0] job id 1120 saw error GenericError("test error")
Sep 22 23:22:50.352 ERRO [0] job id 1120 saw error GenericError("test error")
Sep 22 23:22:50.680 DEBG [1] Read AckReady 1120, : downstairs
Sep 22 23:22:50.681 ERRO [1] job id 1121 saw error GenericError("test error")
Sep 22 23:22:51.012 DEBG up_ds_listen was notified
Sep 22 23:22:51.012 DEBG up_ds_listen process 1119
Sep 22 23:22:51.012 DEBG [A] ack job 1119:120, : downstairs
Sep 22 23:22:51.012 DEBG [rc] retire 1119 clears [JobId(1118), JobId(1119)], : downstairs
Sep 22 23:22:51.012 DEBG up_ds_listen process 1120
Sep 22 23:22:51.012 DEBG [A] ack job 1120:121, : downstairs
Sep 22 23:22:51.065 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:22:51.065 DEBG up_ds_listen was notified
Sep 22 23:22:51.065 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:22:51.066 DEBG IO Read  1122 has deps [JobId(1121)]
Sep 22 23:22:51.079 INFO [lossy] skipping 1122
Sep 22 23:22:51.079 WARN returning error on read!
Sep 22 23:22:51.079 DEBG Read      :1122 deps:[JobId(1121)] res:false
Sep 22 23:22:51.085 DEBG Read      :1122 deps:[JobId(1121)] res:true
Sep 22 23:22:51.106 ERRO [1] job id 1122 saw error GenericError("test error")
Sep 22 23:22:51.108 WARN returning error on flush!
Sep 22 23:22:51.108 DEBG Flush     :1121 extent_limit None deps:[JobId(1120), JobId(1119)] res:false f:49 g:1
Sep 22 23:22:51.109 DEBG Flush     :1121 extent_limit None deps:[JobId(1120), JobId(1119)] res:true f:49 g:1
Sep 22 23:22:51.114 DEBG Read      :1122 deps:[JobId(1121)] res:true
Sep 22 23:22:51.136 WARN returning error on flush!
Sep 22 23:22:51.136 DEBG Flush     :1121 extent_limit None deps:[JobId(1120), JobId(1119)] res:false f:49 g:1
Sep 22 23:22:51.136 DEBG Flush     :1121 extent_limit None deps:[JobId(1120), JobId(1119)] res:true f:49 g:1
Sep 22 23:22:51.142 DEBG Read      :1122 deps:[JobId(1121)] res:true
Sep 22 23:22:51.588 ERRO [0] job id 1121 saw error GenericError("test error")
Sep 22 23:22:51.918 ERRO [2] job id 1121 saw error GenericError("test error")
Sep 22 23:22:51.918 DEBG up_ds_listen was notified
Sep 22 23:22:51.918 DEBG up_ds_listen process 1121
Sep 22 23:22:51.918 DEBG [A] ack job 1121:122, : downstairs
Sep 22 23:22:51.918 DEBG [rc] retire 1121 clears [JobId(1120), JobId(1121)], : downstairs
Sep 22 23:22:51.918 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:51.918 DEBG IO Flush 1123 has deps [JobId(1122)]
Sep 22 23:22:51.920 DEBG Flush     :1123 extent_limit None deps:[JobId(1122)] res:true f:50 g:1
Sep 22 23:22:52.296 DEBG [1] Read AckReady 1122, : downstairs
Sep 22 23:22:52.297 DEBG up_ds_listen was notified
Sep 22 23:22:52.297 DEBG up_ds_listen process 1122
Sep 22 23:22:52.297 DEBG [A] ack job 1122:123, : downstairs
Sep 22 23:22:52.350 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:52.352 DEBG IO Read  1124 has deps [JobId(1123)]
Sep 22 23:22:52.370 DEBG Read      :1124 deps:[JobId(1123)] res:true
Sep 22 23:22:52.393 DEBG Flush     :1123 extent_limit None deps:[JobId(1122)] res:true f:50 g:1
Sep 22 23:22:52.398 DEBG Read      :1124 deps:[JobId(1123)] res:true
Sep 22 23:22:52.420 DEBG Flush     :1123 extent_limit None deps:[JobId(1122)] res:true f:50 g:1
Sep 22 23:22:52.426 DEBG Read      :1124 deps:[JobId(1123)] res:true
Sep 22 23:22:52.543 DEBG IO Flush 1125 has deps [JobId(1124), JobId(1123)]
Sep 22 23:22:53.204 DEBG up_ds_listen was notified
Sep 22 23:22:53.204 DEBG up_ds_listen process 1123
Sep 22 23:22:53.204 DEBG [A] ack job 1123:124, : downstairs
Sep 22 23:22:53.204 DEBG [rc] retire 1123 clears [JobId(1122), JobId(1123)], : downstairs
Sep 22 23:22:53.204 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:53.207 WARN returning error on flush!
Sep 22 23:22:53.207 DEBG Flush     :1125 extent_limit None deps:[JobId(1124), JobId(1123)] res:false f:51 g:1
Sep 22 23:22:53.207 DEBG Flush     :1125 extent_limit None deps:[JobId(1124), JobId(1123)] res:true f:51 g:1
Sep 22 23:22:53.584 DEBG [1] Read AckReady 1124, : downstairs
Sep 22 23:22:53.584 ERRO [1] job id 1125 saw error GenericError("test error")
Sep 22 23:22:53.585 DEBG up_ds_listen was notified
Sep 22 23:22:53.585 DEBG up_ds_listen process 1124
Sep 22 23:22:53.585 DEBG [A] ack job 1124:125, : downstairs
Sep 22 23:22:53.637 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:53.639 DEBG IO Read  1126 has deps [JobId(1125)]
Sep 22 23:22:53.657 DEBG Read      :1126 deps:[JobId(1125)] res:true
Sep 22 23:22:53.680 INFO [lossy] sleeping 1 second
Sep 22 23:22:53.681 DEBG Flush     :1125 extent_limit None deps:[JobId(1124), JobId(1123)] res:true f:51 g:1
Sep 22 23:22:53.681 INFO [lossy] skipping 1126
Sep 22 23:22:53.687 DEBG Read      :1126 deps:[JobId(1125)] res:true
Sep 22 23:22:53.804 DEBG IO Flush 1127 has deps [JobId(1126), JobId(1125)]
Sep 22 23:22:54.466 DEBG up_ds_listen was notified
Sep 22 23:22:54.466 DEBG up_ds_listen process 1125
Sep 22 23:22:54.466 DEBG [A] ack job 1125:126, : downstairs
Sep 22 23:22:54.466 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:54.468 DEBG Flush     :1127 extent_limit None deps:[JobId(1126), JobId(1125)] res:true f:52 g:1
Sep 22 23:22:54.845 DEBG [1] Read AckReady 1126, : downstairs
Sep 22 23:22:54.846 DEBG up_ds_listen was notified
Sep 22 23:22:54.846 DEBG up_ds_listen process 1126
Sep 22 23:22:54.846 DEBG [A] ack job 1126:127, : downstairs
Sep 22 23:22:54.899 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:54.900 WARN returning error on flush!
Sep 22 23:22:54.900 DEBG Flush     :1125 extent_limit None deps:[JobId(1124), JobId(1123)] res:false f:51 g:1
Sep 22 23:22:54.900 INFO [lossy] skipping 1125
Sep 22 23:22:54.900 DEBG Flush     :1125 extent_limit None deps:[JobId(1124), JobId(1123)] res:true f:51 g:1
Sep 22 23:22:54.900 INFO [lossy] sleeping 1 second
Sep 22 23:22:54.900 DEBG IO Read  1128 has deps [JobId(1127)]
Sep 22 23:22:54.901 ERRO [0] job id 1125 saw error GenericError("test error")
Sep 22 23:22:54.901 DEBG [rc] retire 1125 clears [JobId(1124), JobId(1125)], : downstairs
Sep 22 23:22:54.913 INFO [lossy] sleeping 1 second
Sep 22 23:22:54.914 WARN returning error on flush!
Sep 22 23:22:54.915 DEBG Flush     :1127 extent_limit None deps:[JobId(1126), JobId(1125)] res:false f:52 g:1
Sep 22 23:22:54.915 WARN returning error on flush!
Sep 22 23:22:54.915 DEBG Flush     :1127 extent_limit None deps:[JobId(1126), JobId(1125)] res:false f:52 g:1
Sep 22 23:22:54.915 INFO [lossy] skipping 1127
Sep 22 23:22:54.915 WARN returning error on flush!
Sep 22 23:22:54.915 DEBG Flush     :1127 extent_limit None deps:[JobId(1126), JobId(1125)] res:false f:52 g:1
Sep 22 23:22:54.915 DEBG Flush     :1127 extent_limit None deps:[JobId(1126), JobId(1125)] res:true f:52 g:1
Sep 22 23:22:54.921 DEBG Read      :1128 deps:[JobId(1127)] res:true
Sep 22 23:22:55.320 ERRO [2] job id 1127 saw error GenericError("test error")
Sep 22 23:22:55.320 ERRO [2] job id 1127 saw error GenericError("test error")
Sep 22 23:22:55.320 ERRO [2] job id 1127 saw error GenericError("test error")
Sep 22 23:22:55.320 DEBG up_ds_listen was notified
Sep 22 23:22:55.320 DEBG up_ds_listen process 1127
Sep 22 23:22:55.320 DEBG [A] ack job 1127:128, : downstairs
Sep 22 23:22:55.320 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:55.320 DEBG IO Flush 1129 has deps [JobId(1128), JobId(1127)]
Sep 22 23:22:55.323 WARN returning error on flush!
Sep 22 23:22:55.323 DEBG Flush     :1129 extent_limit None deps:[JobId(1128), JobId(1127)] res:false f:53 g:1
Sep 22 23:22:55.323 DEBG Flush     :1129 extent_limit None deps:[JobId(1128), JobId(1127)] res:true f:53 g:1
Sep 22 23:22:55.323 INFO [lossy] sleeping 1 second
Sep 22 23:22:55.699 DEBG [2] Read AckReady 1128, : downstairs
Sep 22 23:22:55.700 ERRO [2] job id 1129 saw error GenericError("test error")
Sep 22 23:22:55.700 DEBG up_ds_listen was notified
Sep 22 23:22:55.700 DEBG up_ds_listen process 1128
Sep 22 23:22:55.700 DEBG [A] ack job 1128:129, : downstairs
Sep 22 23:22:55.753 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:55.754 DEBG IO Read  1130 has deps [JobId(1129)]
Sep 22 23:22:55.821 DEBG IO Flush 1131 has deps [JobId(1130), JobId(1129)]
Sep 22 23:22:55.907 DEBG Read      :1126 deps:[JobId(1125)] res:true
Sep 22 23:22:55.935 DEBG Read      :1128 deps:[JobId(1127)] res:true
Sep 22 23:22:55.958 DEBG Flush     :1127 extent_limit None deps:[JobId(1126), JobId(1125)] res:true f:52 g:1
Sep 22 23:22:55.964 DEBG Read      :1128 deps:[JobId(1127)] res:true
Sep 22 23:22:56.035 DEBG Flush     :1129 extent_limit None deps:[JobId(1128), JobId(1127)] res:true f:53 g:1
Sep 22 23:22:56.040 DEBG Read      :1130 deps:[JobId(1129)] res:true
Sep 22 23:22:56.393 DEBG [rc] retire 1127 clears [JobId(1126), JobId(1127)], : downstairs
Sep 22 23:22:56.446 DEBG Read      :1130 deps:[JobId(1129)] res:true
Sep 22 23:22:56.798 DEBG up_ds_listen was notified
Sep 22 23:22:56.798 DEBG up_ds_listen process 1129
Sep 22 23:22:56.798 DEBG [A] ack job 1129:130, : downstairs
Sep 22 23:22:56.798 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:56.802 INFO [lossy] skipping 1129
Sep 22 23:22:56.802 INFO [lossy] skipping 1130
Sep 22 23:22:56.802 INFO [lossy] skipping 1131
Sep 22 23:22:56.802 DEBG Flush     :1129 extent_limit None deps:[JobId(1128), JobId(1127)] res:true f:53 g:1
Sep 22 23:22:56.808 DEBG Read      :1130 deps:[JobId(1129)] res:true
Sep 22 23:22:56.878 DEBG Flush     :1131 extent_limit None deps:[JobId(1130), JobId(1129)] res:true f:54 g:1
Sep 22 23:22:56.878 INFO [lossy] sleeping 1 second
Sep 22 23:22:57.208 DEBG [rc] retire 1129 clears [JobId(1128), JobId(1129)], : downstairs
Sep 22 23:22:57.256 DEBG Flush     :1131 extent_limit None deps:[JobId(1130), JobId(1129)] res:true f:54 g:1
Sep 22 23:22:57.584 DEBG [1] Read AckReady 1130, : downstairs
Sep 22 23:22:57.585 DEBG up_ds_listen was notified
Sep 22 23:22:57.585 DEBG up_ds_listen process 1130
Sep 22 23:22:57.585 DEBG [A] ack job 1130:131, : downstairs
Sep 22 23:22:57.638 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:58.017 DEBG up_ds_listen was notified
Sep 22 23:22:58.017 DEBG up_ds_listen process 1131
Sep 22 23:22:58.017 DEBG [A] ack job 1131:132, : downstairs
Sep 22 23:22:58.017 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:58.017 DEBG IO Read  1132 has deps [JobId(1131)]
Sep 22 23:22:58.017 INFO [lossy] sleeping 1 second
Sep 22 23:22:58.035 DEBG Read      :1132 deps:[JobId(1131)] res:true
Sep 22 23:22:58.059 DEBG Flush     :1131 extent_limit None deps:[JobId(1130), JobId(1129)] res:true f:54 g:1
Sep 22 23:22:58.064 DEBG Read      :1132 deps:[JobId(1131)] res:true
Sep 22 23:22:58.464 DEBG [rc] retire 1131 clears [JobId(1130), JobId(1131)], : downstairs
Sep 22 23:22:58.841 DEBG [2] Read AckReady 1132, : downstairs
Sep 22 23:22:58.842 DEBG up_ds_listen was notified
Sep 22 23:22:58.842 DEBG up_ds_listen process 1132
Sep 22 23:22:58.842 DEBG [A] ack job 1132:133, : downstairs
Sep 22 23:22:58.895 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:58.896 DEBG IO Flush 1133 has deps [JobId(1132)]
Sep 22 23:22:58.897 DEBG IO Read  1134 has deps [JobId(1133)]
Sep 22 23:22:58.902 DEBG Flush     :1133 extent_limit None deps:[JobId(1132)] res:true f:55 g:1
Sep 22 23:22:58.909 WARN returning error on read!
Sep 22 23:22:58.909 DEBG Read      :1134 deps:[JobId(1133)] res:false
Sep 22 23:22:58.915 DEBG Read      :1134 deps:[JobId(1133)] res:true
Sep 22 23:22:58.936 ERRO [2] job id 1134 saw error GenericError("test error")
Sep 22 23:22:58.938 INFO [lossy] skipping 1133
Sep 22 23:22:58.938 INFO [lossy] skipping 1134
Sep 22 23:22:58.938 INFO [lossy] skipping 1133
Sep 22 23:22:58.938 WARN returning error on flush!
Sep 22 23:22:58.938 DEBG Flush     :1133 extent_limit None deps:[JobId(1132)] res:false f:55 g:1
Sep 22 23:22:58.938 DEBG Flush     :1133 extent_limit None deps:[JobId(1132)] res:true f:55 g:1
Sep 22 23:22:58.943 DEBG Read      :1134 deps:[JobId(1133)] res:true
Sep 22 23:22:59.343 ERRO [0] job id 1133 saw error GenericError("test error")
Sep 22 23:22:59.343 DEBG up_ds_listen was notified
Sep 22 23:22:59.343 DEBG up_ds_listen process 1133
Sep 22 23:22:59.343 DEBG [A] ack job 1133:134, : downstairs
Sep 22 23:22:59.343 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:59.344 INFO [lossy] skipping 1132
Sep 22 23:22:59.344 INFO [lossy] skipping 1134
Sep 22 23:22:59.349 DEBG Read      :1132 deps:[JobId(1131)] res:true
Sep 22 23:22:59.353 DEBG [0] Read AckReady 1000, : downstairs
Sep 22 23:22:59.749 DEBG [2] Read AckReady 1134, : downstairs
Sep 22 23:22:59.750 DEBG up_ds_listen was notified
Sep 22 23:22:59.750 DEBG up_ds_listen process 1134
Sep 22 23:22:59.750 DEBG [A] ack job 1134:135, : downstairs
Sep 22 23:22:59.802 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:22:59.804 DEBG IO Flush 1135 has deps [JobId(1134), JobId(1133)]
Sep 22 23:22:59.805 DEBG IO Read  1136 has deps [JobId(1135)]
Sep 22 23:22:59.810 DEBG Flush     :1135 extent_limit None deps:[JobId(1134), JobId(1133)] res:true f:56 g:1
Sep 22 23:22:59.823 DEBG Read      :1136 deps:[JobId(1135)] res:true
Sep 22 23:22:59.845 DEBG Flush     :1135 extent_limit None deps:[JobId(1134), JobId(1133)] res:true f:56 g:1
Sep 22 23:22:59.845 INFO [lossy] skipping 1136
Sep 22 23:22:59.846 WARN returning error on read!
Sep 22 23:22:59.846 DEBG Read      :1136 deps:[JobId(1135)] res:false
Sep 22 23:22:59.846 INFO [lossy] skipping 1136
Sep 22 23:22:59.846 WARN returning error on read!
Sep 22 23:22:59.846 DEBG Read      :1136 deps:[JobId(1135)] res:false
Sep 22 23:22:59.846 INFO [lossy] skipping 1136
Sep 22 23:22:59.846 INFO [lossy] skipping 1136
Sep 22 23:22:59.851 DEBG Read      :1136 deps:[JobId(1135)] res:true
Sep 22 23:23:00.251 ERRO [0] job id 1136 saw error GenericError("test error")
Sep 22 23:23:00.251 ERRO [0] job id 1136 saw error GenericError("test error")
Sep 22 23:23:00.252 DEBG up_ds_listen was notified
Sep 22 23:23:00.252 DEBG up_ds_listen process 1135
Sep 22 23:23:00.252 DEBG [A] ack job 1135:136, : downstairs
Sep 22 23:23:00.252 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:00.253 INFO [lossy] skipping 1134
Sep 22 23:23:00.253 INFO [lossy] skipping 1134
Sep 22 23:23:00.253 INFO [lossy] skipping 1133
Sep 22 23:23:00.253 INFO [lossy] skipping 1134
Sep 22 23:23:00.253 INFO [lossy] skipping 1133
Sep 22 23:23:00.253 DEBG Flush     :1133 extent_limit None deps:[JobId(1132)] res:true f:55 g:1
Sep 22 23:23:00.253 INFO [lossy] skipping 1134
Sep 22 23:23:00.253 WARN 1135 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:00.253 INFO [lossy] skipping 1136
Sep 22 23:23:00.259 DEBG Read      :1134 deps:[JobId(1133)] res:true
Sep 22 23:23:00.658 DEBG [rc] retire 1133 clears [JobId(1132), JobId(1133)], : downstairs
Sep 22 23:23:00.659 DEBG IO Flush 1137 has deps [JobId(1136), JobId(1135)]
Sep 22 23:23:00.662 INFO [lossy] skipping 1137
Sep 22 23:23:00.662 DEBG Flush     :1137 extent_limit None deps:[JobId(1136), JobId(1135)] res:true f:57 g:1
Sep 22 23:23:01.038 DEBG [2] Read AckReady 1136, : downstairs
Sep 22 23:23:01.039 DEBG up_ds_listen was notified
Sep 22 23:23:01.039 DEBG up_ds_listen process 1136
Sep 22 23:23:01.039 DEBG [A] ack job 1136:137, : downstairs
Sep 22 23:23:01.092 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:01.093 DEBG IO Read  1138 has deps [JobId(1137)]
Sep 22 23:23:01.112 DEBG Read      :1138 deps:[JobId(1137)] res:true
Sep 22 23:23:01.135 WARN returning error on flush!
Sep 22 23:23:01.135 DEBG Flush     :1137 extent_limit None deps:[JobId(1136), JobId(1135)] res:false f:57 g:1
Sep 22 23:23:01.135 INFO [lossy] skipping 1137
Sep 22 23:23:01.135 DEBG Flush     :1137 extent_limit None deps:[JobId(1136), JobId(1135)] res:true f:57 g:1
Sep 22 23:23:01.135 INFO [lossy] sleeping 1 second
Sep 22 23:23:01.136 INFO [lossy] sleeping 1 second
Sep 22 23:23:01.563 ERRO [0] job id 1137 saw error GenericError("test error")
Sep 22 23:23:01.894 DEBG up_ds_listen was notified
Sep 22 23:23:01.894 DEBG up_ds_listen process 1137
Sep 22 23:23:01.894 DEBG [A] ack job 1137:138, : downstairs
Sep 22 23:23:01.894 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:01.894 DEBG IO Flush 1139 has deps [JobId(1138), JobId(1137)]
Sep 22 23:23:01.896 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:true f:58 g:1
Sep 22 23:23:02.273 DEBG [2] Read AckReady 1138, : downstairs
Sep 22 23:23:02.273 DEBG up_ds_listen was notified
Sep 22 23:23:02.273 DEBG up_ds_listen process 1138
Sep 22 23:23:02.274 DEBG [A] ack job 1138:139, : downstairs
Sep 22 23:23:02.326 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:02.333 DEBG Read      :1138 deps:[JobId(1137)] res:true
Sep 22 23:23:02.354 INFO [lossy] skipping 1135
Sep 22 23:23:02.354 INFO [lossy] skipping 1135
Sep 22 23:23:02.354 INFO [lossy] skipping 1135
Sep 22 23:23:02.354 INFO [lossy] skipping 1135
Sep 22 23:23:02.354 DEBG Flush     :1135 extent_limit None deps:[JobId(1134), JobId(1133)] res:true f:56 g:1
Sep 22 23:23:02.354 INFO [lossy] skipping 1136
Sep 22 23:23:02.354 INFO [lossy] skipping 1137
Sep 22 23:23:02.354 INFO [lossy] skipping 1139
Sep 22 23:23:02.360 DEBG Read      :1136 deps:[JobId(1135)] res:true
Sep 22 23:23:02.382 DEBG IO Read  1140 has deps [JobId(1139)]
Sep 22 23:23:02.382 DEBG [rc] retire 1135 clears [JobId(1134), JobId(1135)], : downstairs
Sep 22 23:23:02.394 INFO [lossy] sleeping 1 second
Sep 22 23:23:02.395 DEBG IO Flush 1141 has deps [JobId(1140), JobId(1139)]
Sep 22 23:23:02.397 INFO [lossy] skipping 1139
Sep 22 23:23:02.397 WARN returning error on flush!
Sep 22 23:23:02.397 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:false f:58 g:1
Sep 22 23:23:02.397 INFO [lossy] skipping 1139
Sep 22 23:23:02.397 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:true f:58 g:1
Sep 22 23:23:02.397 INFO [lossy] skipping 1140
Sep 22 23:23:02.397 INFO [lossy] skipping 1141
Sep 22 23:23:02.397 INFO [lossy] skipping 1140
Sep 22 23:23:02.397 INFO [lossy] skipping 1140
Sep 22 23:23:02.403 DEBG Read      :1140 deps:[JobId(1139)] res:true
Sep 22 23:23:02.425 INFO [lossy] skipping 1137
Sep 22 23:23:02.425 INFO [lossy] skipping 1139
Sep 22 23:23:02.425 WARN returning error on flush!
Sep 22 23:23:02.425 DEBG Flush     :1137 extent_limit None deps:[JobId(1136), JobId(1135)] res:false f:57 g:1
Sep 22 23:23:02.425 INFO [lossy] skipping 1139
Sep 22 23:23:02.425 INFO [lossy] skipping 1137
Sep 22 23:23:02.425 WARN returning error on flush!
Sep 22 23:23:02.425 DEBG Flush     :1137 extent_limit None deps:[JobId(1136), JobId(1135)] res:false f:57 g:1
Sep 22 23:23:02.425 WARN returning error on flush!
Sep 22 23:23:02.425 DEBG Flush     :1137 extent_limit None deps:[JobId(1136), JobId(1135)] res:false f:57 g:1
Sep 22 23:23:02.425 DEBG Flush     :1137 extent_limit None deps:[JobId(1136), JobId(1135)] res:true f:57 g:1
Sep 22 23:23:02.431 DEBG Read      :1138 deps:[JobId(1137)] res:true
Sep 22 23:23:02.879 ERRO [0] job id 1139 saw error GenericError("test error")
Sep 22 23:23:03.209 ERRO [1] job id 1137 saw error GenericError("test error")
Sep 22 23:23:03.209 ERRO [1] job id 1137 saw error GenericError("test error")
Sep 22 23:23:03.209 ERRO [1] job id 1137 saw error GenericError("test error")
Sep 22 23:23:03.210 DEBG [rc] retire 1137 clears [JobId(1136), JobId(1137)], : downstairs
Sep 22 23:23:03.210 DEBG up_ds_listen was notified
Sep 22 23:23:03.210 DEBG up_ds_listen process 1139
Sep 22 23:23:03.210 DEBG [A] ack job 1139:140, : downstairs
Sep 22 23:23:03.210 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:03.213 INFO [lossy] sleeping 1 second
Sep 22 23:23:03.214 INFO [lossy] skipping 1139
Sep 22 23:23:03.214 WARN returning error on flush!
Sep 22 23:23:03.214 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:false f:58 g:1
Sep 22 23:23:03.214 DEBG Flush     :1139 extent_limit None deps:[JobId(1138), JobId(1137)] res:true f:58 g:1
Sep 22 23:23:03.214 INFO [lossy] skipping 1140
Sep 22 23:23:03.214 INFO [lossy] skipping 1141
Sep 22 23:23:03.214 INFO [lossy] skipping 1140
Sep 22 23:23:03.214 INFO [lossy] skipping 1141
Sep 22 23:23:03.220 DEBG Read      :1140 deps:[JobId(1139)] res:true
Sep 22 23:23:03.665 DEBG [0] Read AckReady 1140, : downstairs
Sep 22 23:23:03.995 ERRO [1] job id 1139 saw error GenericError("test error")
Sep 22 23:23:03.996 DEBG [rc] retire 1139 clears [JobId(1138), JobId(1139)], : downstairs
Sep 22 23:23:03.996 DEBG up_ds_listen was notified
Sep 22 23:23:03.996 DEBG up_ds_listen process 1140
Sep 22 23:23:03.996 DEBG [A] ack job 1140:141, : downstairs
Sep 22 23:23:04.049 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:04.056 DEBG Read      :1140 deps:[JobId(1139)] res:true
Sep 22 23:23:04.077 DEBG IO Read  1142 has deps [JobId(1141)]
Sep 22 23:23:04.092 DEBG Flush     :1141 extent_limit None deps:[JobId(1140), JobId(1139)] res:true f:59 g:1
Sep 22 23:23:04.097 DEBG Read      :1142 deps:[JobId(1141)] res:true
Sep 22 23:23:04.119 INFO [lossy] skipping 1141
Sep 22 23:23:04.119 DEBG Flush     :1141 extent_limit None deps:[JobId(1140), JobId(1139)] res:true f:59 g:1
Sep 22 23:23:04.119 INFO [lossy] skipping 1142
Sep 22 23:23:04.124 DEBG Read      :1142 deps:[JobId(1141)] res:true
Sep 22 23:23:04.902 DEBG up_ds_listen was notified
Sep 22 23:23:04.902 DEBG up_ds_listen process 1141
Sep 22 23:23:04.902 DEBG [A] ack job 1141:142, : downstairs
Sep 22 23:23:04.902 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:04.902 DEBG Flush     :1141 extent_limit None deps:[JobId(1140), JobId(1139)] res:true f:59 g:1
Sep 22 23:23:04.902 INFO [lossy] skipping 1142
Sep 22 23:23:04.908 DEBG Read      :1142 deps:[JobId(1141)] res:true
Sep 22 23:23:04.929 DEBG IO Flush 1143 has deps [JobId(1142), JobId(1141)]
Sep 22 23:23:04.930 DEBG [rc] retire 1141 clears [JobId(1140), JobId(1141)], : downstairs
Sep 22 23:23:04.932 DEBG Flush     :1143 extent_limit None deps:[JobId(1142), JobId(1141)] res:true f:60 g:1
Sep 22 23:23:04.932 INFO [lossy] sleeping 1 second
Sep 22 23:23:04.934 DEBG Flush     :1143 extent_limit None deps:[JobId(1142), JobId(1141)] res:true f:60 g:1
Sep 22 23:23:05.358 DEBG [1] Read AckReady 1142, : downstairs
Sep 22 23:23:05.688 DEBG [2] Read already AckReady 1142, : downstairs
Sep 22 23:23:05.689 DEBG up_ds_listen was notified
Sep 22 23:23:05.689 DEBG up_ds_listen process 1142
Sep 22 23:23:05.690 DEBG [A] ack job 1142:143, : downstairs
Sep 22 23:23:05.742 DEBG up_ds_listen process 1143
Sep 22 23:23:05.742 DEBG [A] ack job 1143:144, : downstairs
Sep 22 23:23:05.743 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:23:05.743 DEBG up_ds_listen was notified
Sep 22 23:23:05.743 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:23:05.745 INFO [lossy] sleeping 1 second
Sep 22 23:23:05.792 DEBG IO Read  1144 has deps [JobId(1143)]
Sep 22 23:23:06.135 INFO [lossy] skipping 1144
Sep 22 23:23:06.135 INFO [lossy] skipping 1144
Sep 22 23:23:06.135 WARN returning error on read!
Sep 22 23:23:06.135 DEBG Read      :1144 deps:[JobId(1143)] res:false
Sep 22 23:23:06.141 DEBG Read      :1144 deps:[JobId(1143)] res:true
Sep 22 23:23:06.162 INFO [lossy] sleeping 1 second
Sep 22 23:23:06.162 ERRO [1] job id 1144 saw error GenericError("test error")
Sep 22 23:23:06.540 DEBG [1] Read AckReady 1144, : downstairs
Sep 22 23:23:06.541 DEBG up_ds_listen was notified
Sep 22 23:23:06.541 DEBG up_ds_listen process 1144
Sep 22 23:23:06.541 DEBG [A] ack job 1144:145, : downstairs
Sep 22 23:23:06.593 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:06.594 DEBG IO Flush 1145 has deps [JobId(1144), JobId(1143)]
Sep 22 23:23:06.595 DEBG IO Read  1146 has deps [JobId(1145)]
Sep 22 23:23:06.601 INFO [lossy] skipping 1145
Sep 22 23:23:06.601 DEBG Flush     :1145 extent_limit None deps:[JobId(1144), JobId(1143)] res:true f:61 g:1
Sep 22 23:23:06.613 DEBG Read      :1146 deps:[JobId(1145)] res:true
Sep 22 23:23:07.011 DEBG [1] Read AckReady 1146, : downstairs
Sep 22 23:23:07.012 DEBG up_ds_listen was notified
Sep 22 23:23:07.012 DEBG up_ds_listen process 1146
Sep 22 23:23:07.012 DEBG [A] ack job 1146:147, : downstairs
Sep 22 23:23:07.065 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:07.066 DEBG Flush     :1143 extent_limit None deps:[JobId(1142), JobId(1141)] res:true f:60 g:1
Sep 22 23:23:07.072 DEBG Read      :1144 deps:[JobId(1143)] res:true
Sep 22 23:23:07.094 DEBG IO Read  1147 has deps [JobId(1145)]
Sep 22 23:23:07.094 DEBG [rc] retire 1143 clears [JobId(1142), JobId(1143)], : downstairs
Sep 22 23:23:07.106 DEBG IO Flush 1148 has deps [JobId(1147), JobId(1146), JobId(1145)]
Sep 22 23:23:07.106 INFO [lossy] sleeping 1 second
Sep 22 23:23:07.108 WARN returning error on flush!
Sep 22 23:23:07.108 DEBG Flush     :1145 extent_limit None deps:[JobId(1144), JobId(1143)] res:false f:61 g:1
Sep 22 23:23:07.108 INFO [lossy] skipping 1145
Sep 22 23:23:07.108 DEBG Flush     :1145 extent_limit None deps:[JobId(1144), JobId(1143)] res:true f:61 g:1
Sep 22 23:23:07.108 WARN returning error on read!
Sep 22 23:23:07.108 DEBG Read      :1146 deps:[JobId(1145)] res:false
Sep 22 23:23:07.108 WARN returning error on read!
Sep 22 23:23:07.108 DEBG Read      :1147 deps:[JobId(1145)] res:false
Sep 22 23:23:07.114 DEBG Read      :1146 deps:[JobId(1145)] res:true
Sep 22 23:23:07.513 ERRO [0] job id 1145 saw error GenericError("test error")
Sep 22 23:23:07.513 ERRO [0] job id 1146 saw error GenericError("test error")
Sep 22 23:23:07.513 ERRO [0] job id 1147 saw error GenericError("test error")
Sep 22 23:23:07.513 DEBG up_ds_listen was notified
Sep 22 23:23:07.513 DEBG up_ds_listen process 1145
Sep 22 23:23:07.513 DEBG [A] ack job 1145:146, : downstairs
Sep 22 23:23:07.513 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:07.513 WARN returning error on read!
Sep 22 23:23:07.513 DEBG Read      :1144 deps:[JobId(1143)] res:false
Sep 22 23:23:07.513 INFO [lossy] skipping 1147
Sep 22 23:23:07.519 DEBG Read      :1144 deps:[JobId(1143)] res:true
Sep 22 23:23:07.541 ERRO [2] job id 1144 saw error GenericError("test error")
Sep 22 23:23:07.543 WARN returning error on read!
Sep 22 23:23:07.543 DEBG Read      :1147 deps:[JobId(1145)] res:false
Sep 22 23:23:07.549 DEBG Read      :1147 deps:[JobId(1145)] res:true
Sep 22 23:23:07.949 ERRO [0] job id 1147 saw error GenericError("test error")
Sep 22 23:23:07.950 INFO [lossy] sleeping 1 second
Sep 22 23:23:08.333 DEBG Read      :1147 deps:[JobId(1145)] res:true
Sep 22 23:23:08.357 INFO [lossy] sleeping 1 second
Sep 22 23:23:08.732 DEBG [0] Read AckReady 1147, : downstairs
Sep 22 23:23:08.733 DEBG up_ds_listen was notified
Sep 22 23:23:08.733 DEBG up_ds_listen process 1147
Sep 22 23:23:08.733 DEBG [A] ack job 1147:148, : downstairs
Sep 22 23:23:08.786 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:08.788 DEBG Flush     :1148 extent_limit None deps:[JobId(1147), JobId(1146), JobId(1145)] res:true f:62 g:1
Sep 22 23:23:08.836 DEBG IO Read  1149 has deps [JobId(1148)]
Sep 22 23:23:09.179 DEBG Flush     :1145 extent_limit None deps:[JobId(1144), JobId(1143)] res:true f:61 g:1
Sep 22 23:23:09.184 DEBG Read      :1146 deps:[JobId(1145)] res:true
Sep 22 23:23:09.206 INFO [lossy] sleeping 1 second
Sep 22 23:23:09.206 DEBG [rc] retire 1145 clears [JobId(1144), JobId(1145)], : downstairs
Sep 22 23:23:09.213 DEBG Read      :1147 deps:[JobId(1145)] res:true
Sep 22 23:23:09.613 DEBG IO Flush 1150 has deps [JobId(1149), JobId(1148)]
Sep 22 23:23:09.613 INFO [lossy] skipping 1148
Sep 22 23:23:09.613 DEBG Flush     :1148 extent_limit None deps:[JobId(1147), JobId(1146), JobId(1145)] res:true f:62 g:1
Sep 22 23:23:09.613 INFO [lossy] skipping 1149
Sep 22 23:23:09.619 DEBG Read      :1149 deps:[JobId(1148)] res:true
Sep 22 23:23:09.641 DEBG up_ds_listen was notified
Sep 22 23:23:09.641 DEBG up_ds_listen process 1148
Sep 22 23:23:09.641 DEBG [A] ack job 1148:149, : downstairs
Sep 22 23:23:09.641 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:09.643 DEBG Flush     :1148 extent_limit None deps:[JobId(1147), JobId(1146), JobId(1145)] res:true f:62 g:1
Sep 22 23:23:09.643 WARN returning error on read!
Sep 22 23:23:09.643 DEBG Read      :1149 deps:[JobId(1148)] res:false
Sep 22 23:23:09.649 DEBG Read      :1149 deps:[JobId(1148)] res:true
Sep 22 23:23:10.049 DEBG [rc] retire 1148 clears [JobId(1146), JobId(1147), JobId(1148)], : downstairs
Sep 22 23:23:10.049 ERRO [2] job id 1149 saw error GenericError("test error")
Sep 22 23:23:10.051 INFO [lossy] skipping 1150
Sep 22 23:23:10.051 DEBG Flush     :1150 extent_limit None deps:[JobId(1149), JobId(1148)] res:true f:63 g:1
Sep 22 23:23:10.051 INFO [lossy] sleeping 1 second
Sep 22 23:23:10.428 DEBG [0] Read AckReady 1149, : downstairs
Sep 22 23:23:10.428 DEBG up_ds_listen was notified
Sep 22 23:23:10.428 DEBG up_ds_listen process 1149
Sep 22 23:23:10.428 DEBG [A] ack job 1149:150, : downstairs
Sep 22 23:23:10.481 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:10.483 INFO [lossy] skipping 1149
Sep 22 23:23:10.489 DEBG Read      :1149 deps:[JobId(1148)] res:true
Sep 22 23:23:10.510 DEBG IO Read  1151 has deps [JobId(1150)]
Sep 22 23:23:10.524 DEBG Flush     :1150 extent_limit None deps:[JobId(1149), JobId(1148)] res:true f:63 g:1
Sep 22 23:23:10.530 DEBG Read      :1151 deps:[JobId(1150)] res:true
Sep 22 23:23:10.930 DEBG up_ds_listen was notified
Sep 22 23:23:10.930 DEBG up_ds_listen process 1150
Sep 22 23:23:10.930 DEBG [A] ack job 1150:151, : downstairs
Sep 22 23:23:10.930 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:10.932 DEBG Flush     :1150 extent_limit None deps:[JobId(1149), JobId(1148)] res:true f:63 g:1
Sep 22 23:23:10.937 DEBG Read      :1151 deps:[JobId(1150)] res:true
Sep 22 23:23:11.338 DEBG [rc] retire 1150 clears [JobId(1149), JobId(1150)], : downstairs
Sep 22 23:23:11.338 DEBG IO Flush 1152 has deps [JobId(1151)]
Sep 22 23:23:11.338 WARN returning error on read!
Sep 22 23:23:11.338 DEBG Read      :1151 deps:[JobId(1150)] res:false
Sep 22 23:23:11.338 WARN returning error on read!
Sep 22 23:23:11.338 DEBG Read      :1151 deps:[JobId(1150)] res:false
Sep 22 23:23:11.338 INFO [lossy] skipping 1151
Sep 22 23:23:11.338 WARN returning error on read!
Sep 22 23:23:11.338 DEBG Read      :1151 deps:[JobId(1150)] res:false
Sep 22 23:23:11.339 WARN returning error on read!
Sep 22 23:23:11.339 DEBG Read      :1151 deps:[JobId(1150)] res:false
Sep 22 23:23:11.339 INFO [lossy] skipping 1151
Sep 22 23:23:11.339 WARN returning error on read!
Sep 22 23:23:11.339 DEBG Read      :1151 deps:[JobId(1150)] res:false
Sep 22 23:23:11.339 INFO [lossy] skipping 1151
Sep 22 23:23:11.344 DEBG Read      :1151 deps:[JobId(1150)] res:true
Sep 22 23:23:11.366 ERRO [0] job id 1151 saw error GenericError("test error")
Sep 22 23:23:11.366 ERRO [0] job id 1151 saw error GenericError("test error")
Sep 22 23:23:11.366 ERRO [0] job id 1151 saw error GenericError("test error")
Sep 22 23:23:11.366 ERRO [0] job id 1151 saw error GenericError("test error")
Sep 22 23:23:11.366 ERRO [0] job id 1151 saw error GenericError("test error")
Sep 22 23:23:11.369 INFO [lossy] sleeping 1 second
Sep 22 23:23:11.746 DEBG [2] Read AckReady 1151, : downstairs
Sep 22 23:23:11.747 DEBG up_ds_listen was notified
Sep 22 23:23:11.747 DEBG up_ds_listen process 1151
Sep 22 23:23:11.747 DEBG [A] ack job 1151:152, : downstairs
Sep 22 23:23:11.799 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:11.802 INFO [lossy] sleeping 1 second
Sep 22 23:23:11.849 DEBG IO Read  1153 has deps [JobId(1152)]
Sep 22 23:23:12.187 INFO [lossy] sleeping 1 second
Sep 22 23:23:12.194 DEBG IO Flush 1154 has deps [JobId(1153), JobId(1152)]
Sep 22 23:23:12.571 INFO [lossy] skipping 1152
Sep 22 23:23:12.571 INFO [lossy] skipping 1154
Sep 22 23:23:12.571 INFO [lossy] skipping 1152
Sep 22 23:23:12.571 INFO [lossy] skipping 1154
Sep 22 23:23:12.571 DEBG Flush     :1152 extent_limit None deps:[JobId(1151)] res:true f:64 g:1
Sep 22 23:23:12.571 INFO [lossy] skipping 1153
Sep 22 23:23:12.572 WARN returning error on read!
Sep 22 23:23:12.572 DEBG Read      :1153 deps:[JobId(1152)] res:false
Sep 22 23:23:12.572 INFO [lossy] skipping 1153
Sep 22 23:23:12.572 INFO [lossy] skipping 1153
Sep 22 23:23:12.577 DEBG Read      :1153 deps:[JobId(1152)] res:true
Sep 22 23:23:12.599 ERRO [2] job id 1153 saw error GenericError("test error")
Sep 22 23:23:12.600 WARN returning error on flush!
Sep 22 23:23:12.600 DEBG Flush     :1154 extent_limit None deps:[JobId(1153), JobId(1152)] res:false f:65 g:1
Sep 22 23:23:12.600 INFO [lossy] skipping 1154
Sep 22 23:23:12.600 DEBG Flush     :1154 extent_limit None deps:[JobId(1153), JobId(1152)] res:true f:65 g:1
Sep 22 23:23:12.600 INFO [lossy] sleeping 1 second
Sep 22 23:23:12.976 DEBG [2] Read AckReady 1153, : downstairs
Sep 22 23:23:12.977 ERRO [2] job id 1154 saw error GenericError("test error")
Sep 22 23:23:12.977 DEBG up_ds_listen was notified
Sep 22 23:23:12.977 DEBG up_ds_listen process 1153
Sep 22 23:23:12.977 DEBG [A] ack job 1153:154, : downstairs
Sep 22 23:23:13.030 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:13.031 DEBG Flush     :1152 extent_limit None deps:[JobId(1151)] res:true f:64 g:1
Sep 22 23:23:13.037 DEBG Read      :1153 deps:[JobId(1152)] res:true
Sep 22 23:23:13.058 DEBG IO Read  1155 has deps [JobId(1154)]
Sep 22 23:23:13.064 DEBG up_ds_listen was notified
Sep 22 23:23:13.064 DEBG up_ds_listen process 1152
Sep 22 23:23:13.064 DEBG [A] ack job 1152:153, : downstairs
Sep 22 23:23:13.064 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:13.072 INFO [lossy] skipping 1154
Sep 22 23:23:13.072 DEBG Flush     :1154 extent_limit None deps:[JobId(1153), JobId(1152)] res:true f:65 g:1
Sep 22 23:23:13.072 INFO [lossy] sleeping 1 second
Sep 22 23:23:13.448 DEBG up_ds_listen was notified
Sep 22 23:23:13.448 DEBG up_ds_listen process 1154
Sep 22 23:23:13.448 DEBG [A] ack job 1154:155, : downstairs
Sep 22 23:23:13.448 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:13.449 INFO [lossy] skipping 1152
Sep 22 23:23:13.449 INFO [lossy] skipping 1154
Sep 22 23:23:13.449 DEBG Flush     :1152 extent_limit None deps:[JobId(1151)] res:true f:64 g:1
Sep 22 23:23:13.449 INFO [lossy] skipping 1153
Sep 22 23:23:13.449 INFO [lossy] skipping 1155
Sep 22 23:23:13.455 DEBG Read      :1153 deps:[JobId(1152)] res:true
Sep 22 23:23:13.476 DEBG [rc] retire 1152 clears [JobId(1151), JobId(1152)], : downstairs
Sep 22 23:23:13.477 INFO [lossy] skipping 1155
Sep 22 23:23:13.477 INFO [lossy] skipping 1155
Sep 22 23:23:13.477 DEBG Flush     :1154 extent_limit None deps:[JobId(1153), JobId(1152)] res:true f:65 g:1
Sep 22 23:23:13.483 DEBG Read      :1155 deps:[JobId(1154)] res:true
Sep 22 23:23:13.882 DEBG [rc] retire 1154 clears [JobId(1153), JobId(1154)], : downstairs
Sep 22 23:23:13.883 DEBG IO Flush 1156 has deps [JobId(1155)]
Sep 22 23:23:13.883 INFO [lossy] skipping 1155
Sep 22 23:23:13.888 DEBG Read      :1155 deps:[JobId(1154)] res:true
Sep 22 23:23:13.912 INFO [lossy] sleeping 1 second
Sep 22 23:23:14.288 DEBG [0] Read AckReady 1155, : downstairs
Sep 22 23:23:14.289 DEBG up_ds_listen was notified
Sep 22 23:23:14.289 DEBG up_ds_listen process 1155
Sep 22 23:23:14.289 DEBG [A] ack job 1155:156, : downstairs
Sep 22 23:23:14.342 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:14.344 DEBG Flush     :1156 extent_limit None deps:[JobId(1155)] res:true f:66 g:1
Sep 22 23:23:14.344 INFO [lossy] skipping 1155
Sep 22 23:23:14.344 WARN returning error on read!
Sep 22 23:23:14.344 DEBG Read      :1155 deps:[JobId(1154)] res:false
Sep 22 23:23:14.350 DEBG Read      :1155 deps:[JobId(1154)] res:true
Sep 22 23:23:14.371 INFO [lossy] sleeping 1 second
Sep 22 23:23:14.419 DEBG IO Read  1157 has deps [JobId(1156)]
Sep 22 23:23:14.420 ERRO [1] job id 1155 saw error GenericError("test error")
Sep 22 23:23:14.762 DEBG IO Flush 1158 has deps [JobId(1157), JobId(1156)]
Sep 22 23:23:14.765 INFO [lossy] sleeping 1 second
Sep 22 23:23:14.849 DEBG [1] Read already AckReady 1000, : downstairs
Sep 22 23:23:15.143 DEBG Flush     :1156 extent_limit None deps:[JobId(1155)] res:true f:66 g:1
Sep 22 23:23:15.143 INFO [lossy] skipping 1157
Sep 22 23:23:15.143 INFO [lossy] skipping 1158
Sep 22 23:23:15.149 DEBG Read      :1157 deps:[JobId(1156)] res:true
Sep 22 23:23:15.170 DEBG up_ds_listen was notified
Sep 22 23:23:15.170 DEBG up_ds_listen process 1156
Sep 22 23:23:15.170 DEBG [A] ack job 1156:157, : downstairs
Sep 22 23:23:15.170 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:15.171 DEBG Flush     :1158 extent_limit None deps:[JobId(1157), JobId(1156)] res:true f:67 g:1
Sep 22 23:23:15.172 INFO [lossy] sleeping 1 second
Sep 22 23:23:15.547 DEBG [0] Read AckReady 1157, : downstairs
Sep 22 23:23:15.547 DEBG up_ds_listen was notified
Sep 22 23:23:15.547 DEBG up_ds_listen process 1157
Sep 22 23:23:15.547 DEBG [A] ack job 1157:158, : downstairs
Sep 22 23:23:15.600 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:15.607 DEBG Read      :1157 deps:[JobId(1156)] res:true
Sep 22 23:23:15.628 DEBG IO Read  1159 has deps [JobId(1158)]
Sep 22 23:23:15.642 INFO [lossy] skipping 1158
Sep 22 23:23:15.642 DEBG Flush     :1158 extent_limit None deps:[JobId(1157), JobId(1156)] res:true f:67 g:1
Sep 22 23:23:15.647 DEBG Read      :1159 deps:[JobId(1158)] res:true
Sep 22 23:23:16.047 DEBG up_ds_listen was notified
Sep 22 23:23:16.047 DEBG up_ds_listen process 1158
Sep 22 23:23:16.047 DEBG [A] ack job 1158:159, : downstairs
Sep 22 23:23:16.047 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:16.047 DEBG Flush     :1156 extent_limit None deps:[JobId(1155)] res:true f:66 g:1
Sep 22 23:23:16.053 DEBG Read      :1157 deps:[JobId(1156)] res:true
Sep 22 23:23:16.075 DEBG [rc] retire 1156 clears [JobId(1155), JobId(1156)], : downstairs
Sep 22 23:23:16.453 DEBG [2] Read AckReady 1159, : downstairs
Sep 22 23:23:16.454 DEBG up_ds_listen was notified
Sep 22 23:23:16.454 DEBG up_ds_listen process 1159
Sep 22 23:23:16.454 DEBG [A] ack job 1159:160, : downstairs
Sep 22 23:23:16.507 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:16.509 DEBG Flush     :1158 extent_limit None deps:[JobId(1157), JobId(1156)] res:true f:67 g:1
Sep 22 23:23:16.509 WARN returning error on read!
Sep 22 23:23:16.509 DEBG Read      :1159 deps:[JobId(1158)] res:false
Sep 22 23:23:16.509 INFO [lossy] skipping 1159
Sep 22 23:23:16.509 INFO [lossy] skipping 1159
Sep 22 23:23:16.515 DEBG Read      :1159 deps:[JobId(1158)] res:true
Sep 22 23:23:16.536 DEBG IO Flush 1160 has deps [JobId(1159), JobId(1158)]
Sep 22 23:23:16.542 DEBG Read      :1159 deps:[JobId(1158)] res:true
Sep 22 23:23:16.612 DEBG IO Read  1161 has deps [JobId(1160)]
Sep 22 23:23:16.942 DEBG [rc] retire 1158 clears [JobId(1157), JobId(1158)], : downstairs
Sep 22 23:23:16.942 ERRO [1] job id 1159 saw error GenericError("test error")
Sep 22 23:23:16.948 WARN returning error on flush!
Sep 22 23:23:16.948 DEBG Flush     :1160 extent_limit None deps:[JobId(1159), JobId(1158)] res:false f:68 g:1
Sep 22 23:23:16.948 INFO [lossy] skipping 1160
Sep 22 23:23:16.948 INFO [lossy] skipping 1160
Sep 22 23:23:16.948 DEBG Flush     :1160 extent_limit None deps:[JobId(1159), JobId(1158)] res:true f:68 g:1
Sep 22 23:23:16.955 ERRO [2] job id 1160 saw error GenericError("test error")
Sep 22 23:23:16.955 INFO [lossy] skipping 1161
Sep 22 23:23:16.961 DEBG Read      :1161 deps:[JobId(1160)] res:true
Sep 22 23:23:16.985 INFO [lossy] sleeping 1 second
Sep 22 23:23:16.986 DEBG Flush     :1160 extent_limit None deps:[JobId(1159), JobId(1158)] res:true f:68 g:1
Sep 22 23:23:16.992 DEBG Read      :1161 deps:[JobId(1160)] res:true
Sep 22 23:23:17.770 DEBG up_ds_listen was notified
Sep 22 23:23:17.770 DEBG up_ds_listen process 1160
Sep 22 23:23:17.770 DEBG [A] ack job 1160:161, : downstairs
Sep 22 23:23:17.770 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:17.772 DEBG IO Flush 1162 has deps [JobId(1161), JobId(1160)]
Sep 22 23:23:18.148 DEBG [2] Read AckReady 1161, : downstairs
Sep 22 23:23:18.149 DEBG up_ds_listen was notified
Sep 22 23:23:18.149 DEBG up_ds_listen process 1161
Sep 22 23:23:18.149 DEBG [A] ack job 1161:162, : downstairs
Sep 22 23:23:18.201 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:18.201 DEBG Flush     :1162 extent_limit None deps:[JobId(1161), JobId(1160)] res:true f:69 g:1
Sep 22 23:23:19.989 INFO [lossy] skipping 1160
Sep 22 23:23:19.989 DEBG Flush     :1160 extent_limit None deps:[JobId(1159), JobId(1158)] res:true f:68 g:1
Sep 22 23:23:19.989 INFO [lossy] sleeping 1 second
Sep 22 23:23:20.318 DEBG IO Write 1163 has deps [JobId(1162), JobId(1160)]
Sep 22 23:23:20.318 DEBG IO Flush 1164 has deps [JobId(1163), JobId(1162)]
Sep 22 23:23:20.319 DEBG [rc] retire 1160 clears [JobId(1159), JobId(1160)], : downstairs
Sep 22 23:23:20.319 DEBG up_ds_listen was notified
Sep 22 23:23:20.319 DEBG up_ds_listen process 1163
Sep 22 23:23:20.319 DEBG [A] ack job 1163:164, : downstairs
Sep 22 23:23:20.319 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:20.711 DEBG IO Write 1165 has deps [JobId(1164), JobId(1162)]
Sep 22 23:23:20.711 DEBG up_ds_listen was notified
Sep 22 23:23:20.711 DEBG up_ds_listen process 1165
Sep 22 23:23:20.711 DEBG [A] ack job 1165:166, : downstairs
Sep 22 23:23:20.711 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:21.041 DEBG IO Write 1166 has deps [JobId(1164), JobId(1162)]
Sep 22 23:23:21.041 DEBG up_ds_listen was notified
Sep 22 23:23:21.041 DEBG up_ds_listen process 1166
Sep 22 23:23:21.041 DEBG [A] ack job 1166:167, : downstairs
Sep 22 23:23:21.041 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:21.042 DEBG IO Flush 1167 has deps [JobId(1166), JobId(1165), JobId(1164)]
Sep 22 23:23:21.042 INFO [lossy] skipping 1161
Sep 22 23:23:21.042 INFO [lossy] skipping 1162
Sep 22 23:23:21.042 INFO [lossy] skipping 1161
Sep 22 23:23:21.042 INFO [lossy] skipping 1162
Sep 22 23:23:21.048 DEBG Read      :1161 deps:[JobId(1160)] res:true
Sep 22 23:23:21.399 DEBG IO Write 1168 has deps [JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:21.399 DEBG up_ds_listen was notified
Sep 22 23:23:21.399 DEBG up_ds_listen process 1168
Sep 22 23:23:21.399 DEBG [A] ack job 1168:169, : downstairs
Sep 22 23:23:21.399 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:21.730 DEBG IO Write 1169 has deps [JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:21.730 DEBG up_ds_listen was notified
Sep 22 23:23:21.730 DEBG up_ds_listen process 1169
Sep 22 23:23:21.730 DEBG [A] ack job 1169:170, : downstairs
Sep 22 23:23:21.730 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:21.731 DEBG IO Flush 1170 has deps [JobId(1169), JobId(1168), JobId(1167)]
Sep 22 23:23:22.060 DEBG IO Write 1171 has deps [JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:22.061 DEBG up_ds_listen was notified
Sep 22 23:23:22.061 DEBG up_ds_listen process 1171
Sep 22 23:23:22.061 DEBG [A] ack job 1171:172, : downstairs
Sep 22 23:23:22.061 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:22.391 DEBG IO Write 1172 has deps [JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:22.391 DEBG up_ds_listen was notified
Sep 22 23:23:22.391 DEBG up_ds_listen process 1172
Sep 22 23:23:22.391 DEBG [A] ack job 1172:173, : downstairs
Sep 22 23:23:22.391 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:22.392 DEBG IO Flush 1173 has deps [JobId(1172), JobId(1171), JobId(1170)]
Sep 22 23:23:22.721 DEBG IO Write 1174 has deps [JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:22.722 DEBG up_ds_listen was notified
Sep 22 23:23:22.722 DEBG up_ds_listen process 1174
Sep 22 23:23:22.722 DEBG [A] ack job 1174:175, : downstairs
Sep 22 23:23:22.722 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:23.052 DEBG IO Write 1175 has deps [JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:23.052 DEBG up_ds_listen was notified
Sep 22 23:23:23.052 DEBG up_ds_listen process 1175
Sep 22 23:23:23.052 DEBG [A] ack job 1175:176, : downstairs
Sep 22 23:23:23.052 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:23.053 DEBG IO Flush 1176 has deps [JobId(1175), JobId(1174), JobId(1173)]
Sep 22 23:23:23.446 DEBG IO Write 1177 has deps [JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:23.446 DEBG up_ds_listen was notified
Sep 22 23:23:23.446 DEBG up_ds_listen process 1177
Sep 22 23:23:23.446 DEBG [A] ack job 1177:178, : downstairs
Sep 22 23:23:23.446 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:23.587 INFO [lossy] skipping 1163
Sep 22 23:23:23.588 WARN returning error on write!
Sep 22 23:23:23.589 DEBG Write     :1163 deps:[JobId(1162), JobId(1160)] res:false
Sep 22 23:23:23.619 DEBG Write     :1163 deps:[JobId(1162), JobId(1160)] res:true
Sep 22 23:23:23.624 DEBG Flush     :1164 extent_limit None deps:[JobId(1163), JobId(1162)] res:true f:70 g:1
Sep 22 23:23:23.624 DEBG IO Flush 1178 has deps [JobId(1177), JobId(1176)]
Sep 22 23:23:23.953 DEBG IO Write 1179 has deps [JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:23.953 DEBG up_ds_listen was notified
Sep 22 23:23:23.953 DEBG up_ds_listen process 1179
Sep 22 23:23:23.953 DEBG [A] ack job 1179:180, : downstairs
Sep 22 23:23:23.953 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:24.283 DEBG IO Write 1180 has deps [JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:24.283 DEBG up_ds_listen was notified
Sep 22 23:23:24.283 DEBG up_ds_listen process 1180
Sep 22 23:23:24.283 DEBG [A] ack job 1180:181, : downstairs
Sep 22 23:23:24.283 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:24.284 DEBG IO Flush 1181 has deps [JobId(1180), JobId(1179), JobId(1178)]
Sep 22 23:23:24.613 DEBG IO Write 1182 has deps [JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:24.613 DEBG up_ds_listen was notified
Sep 22 23:23:24.613 DEBG up_ds_listen process 1182
Sep 22 23:23:24.613 DEBG [A] ack job 1182:183, : downstairs
Sep 22 23:23:24.613 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:24.943 DEBG IO Write 1183 has deps [JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:24.943 DEBG up_ds_listen was notified
Sep 22 23:23:24.943 DEBG up_ds_listen process 1183
Sep 22 23:23:24.943 DEBG [A] ack job 1183:184, : downstairs
Sep 22 23:23:24.943 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:24.944 DEBG IO Flush 1184 has deps [JobId(1183), JobId(1182), JobId(1181)]
Sep 22 23:23:25.273 DEBG IO Write 1185 has deps [JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:25.273 DEBG up_ds_listen was notified
Sep 22 23:23:25.273 DEBG up_ds_listen process 1185
Sep 22 23:23:25.273 DEBG [A] ack job 1185:186, : downstairs
Sep 22 23:23:25.274 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:25.603 DEBG IO Write 1186 has deps [JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:25.603 DEBG up_ds_listen was notified
Sep 22 23:23:25.604 DEBG up_ds_listen process 1186
Sep 22 23:23:25.604 DEBG [A] ack job 1186:187, : downstairs
Sep 22 23:23:25.604 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:25.604 DEBG IO Flush 1187 has deps [JobId(1186), JobId(1185), JobId(1184)]
Sep 22 23:23:25.604 INFO [lossy] sleeping 1 second
Sep 22 23:23:25.933 DEBG IO Write 1188 has deps [JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:25.934 DEBG up_ds_listen was notified
Sep 22 23:23:25.934 DEBG up_ds_listen process 1188
Sep 22 23:23:25.934 DEBG [A] ack job 1188:189, : downstairs
Sep 22 23:23:25.934 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:26.264 DEBG IO Write 1189 has deps [JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:26.264 DEBG up_ds_listen was notified
Sep 22 23:23:26.264 DEBG up_ds_listen process 1189
Sep 22 23:23:26.264 DEBG [A] ack job 1189:190, : downstairs
Sep 22 23:23:26.264 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:26.265 DEBG IO Flush 1190 has deps [JobId(1189), JobId(1188), JobId(1187)]
Sep 22 23:23:26.657 DEBG IO Write 1191 has deps [JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:26.657 DEBG up_ds_listen was notified
Sep 22 23:23:26.657 DEBG up_ds_listen process 1191
Sep 22 23:23:26.657 DEBG [A] ack job 1191:192, : downstairs
Sep 22 23:23:26.657 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:26.828 DEBG Write     :1165 deps:[JobId(1164), JobId(1162)] res:true
Sep 22 23:23:26.829 DEBG IO Flush 1192 has deps [JobId(1191), JobId(1190)]
Sep 22 23:23:27.158 DEBG IO Write 1193 has deps [JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:27.159 DEBG up_ds_listen was notified
Sep 22 23:23:27.159 DEBG up_ds_listen process 1193
Sep 22 23:23:27.159 DEBG [A] ack job 1193:194, : downstairs
Sep 22 23:23:27.159 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:27.488 DEBG IO Write 1194 has deps [JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:27.489 DEBG up_ds_listen was notified
Sep 22 23:23:27.489 DEBG up_ds_listen process 1194
Sep 22 23:23:27.489 DEBG [A] ack job 1194:195, : downstairs
Sep 22 23:23:27.489 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:27.489 DEBG IO Flush 1195 has deps [JobId(1194), JobId(1193), JobId(1192)]
Sep 22 23:23:27.819 DEBG IO Write 1196 has deps [JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:27.819 DEBG up_ds_listen was notified
Sep 22 23:23:27.819 DEBG up_ds_listen process 1196
Sep 22 23:23:27.819 DEBG [A] ack job 1196:197, : downstairs
Sep 22 23:23:27.819 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:28.150 DEBG IO Write 1197 has deps [JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:28.150 DEBG up_ds_listen was notified
Sep 22 23:23:28.150 DEBG up_ds_listen process 1197
Sep 22 23:23:28.150 DEBG [A] ack job 1197:198, : downstairs
Sep 22 23:23:28.150 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:28.151 DEBG IO Flush 1198 has deps [JobId(1197), JobId(1196), JobId(1195)]
Sep 22 23:23:28.480 DEBG IO Write 1199 has deps [JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:28.480 DEBG up_ds_listen was notified
Sep 22 23:23:28.480 DEBG up_ds_listen process 1199
Sep 22 23:23:28.481 DEBG [A] ack job 1199:200, : downstairs
Sep 22 23:23:28.481 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:28.811 DEBG IO Write 1200 has deps [JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:28.811 DEBG up_ds_listen was notified
Sep 22 23:23:28.811 DEBG up_ds_listen process 1200
Sep 22 23:23:28.811 DEBG [A] ack job 1200:201, : downstairs
Sep 22 23:23:28.811 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:28.812 DEBG IO Flush 1201 has deps [JobId(1200), JobId(1199), JobId(1198)]
Sep 22 23:23:29.141 DEBG IO Write 1202 has deps [JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:29.141 DEBG up_ds_listen was notified
Sep 22 23:23:29.142 DEBG up_ds_listen process 1202
Sep 22 23:23:29.142 DEBG [A] ack job 1202:203, : downstairs
Sep 22 23:23:29.142 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:29.472 DEBG IO Write 1203 has deps [JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:29.472 DEBG up_ds_listen was notified
Sep 22 23:23:29.472 DEBG up_ds_listen process 1203
Sep 22 23:23:29.472 DEBG [A] ack job 1203:204, : downstairs
Sep 22 23:23:29.472 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:29.473 DEBG IO Flush 1204 has deps [JobId(1203), JobId(1202), JobId(1201)]
Sep 22 23:23:29.865 DEBG IO Write 1205 has deps [JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:29.865 DEBG up_ds_listen was notified
Sep 22 23:23:29.865 DEBG up_ds_listen process 1205
Sep 22 23:23:29.865 DEBG [A] ack job 1205:206, : downstairs
Sep 22 23:23:29.865 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:30.005 INFO [lossy] skipping 1166
Sep 22 23:23:30.006 INFO [lossy] skipping 1167
Sep 22 23:23:30.037 DEBG Write     :1166 deps:[JobId(1164), JobId(1162)] res:true
Sep 22 23:23:30.038 INFO [lossy] skipping 1167
Sep 22 23:23:30.046 DEBG Flush     :1167 extent_limit None deps:[JobId(1166), JobId(1165), JobId(1164)] res:true f:71 g:1
Sep 22 23:23:30.047 DEBG IO Flush 1206 has deps [JobId(1205), JobId(1204)]
Sep 22 23:23:30.376 DEBG IO Write 1207 has deps [JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:30.376 DEBG up_ds_listen was notified
Sep 22 23:23:30.376 DEBG up_ds_listen process 1207
Sep 22 23:23:30.376 DEBG [A] ack job 1207:208, : downstairs
Sep 22 23:23:30.376 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:30.406 DEBG [2] Read already AckReady 1000, : downstairs
Sep 22 23:23:30.520 DEBG up_ds_listen was notified
Sep 22 23:23:30.520 DEBG up_ds_listen process 1000
Sep 22 23:23:30.523 DEBG [A] ack job 1000:1, : downstairs
Sep 22 23:23:30.706 DEBG IO Write 1208 has deps [JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:30.706 DEBG up_ds_listen was notified
Sep 22 23:23:30.706 DEBG up_ds_listen process 1208
Sep 22 23:23:30.706 DEBG [A] ack job 1208:209, : downstairs
Sep 22 23:23:30.707 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:30.707 DEBG IO Flush 1209 has deps [JobId(1208), JobId(1207), JobId(1206)]
Sep 22 23:23:31.037 DEBG IO Write 1210 has deps [JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:31.037 DEBG up_ds_listen was notified
Sep 22 23:23:31.037 DEBG up_ds_listen process 1210
Sep 22 23:23:31.037 DEBG [A] ack job 1210:211, : downstairs
Sep 22 23:23:31.037 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:31.367 DEBG IO Write 1211 has deps [JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:31.367 DEBG up_ds_listen was notified
Sep 22 23:23:31.367 DEBG up_ds_listen process 1211
Sep 22 23:23:31.367 DEBG [A] ack job 1211:212, : downstairs
Sep 22 23:23:31.367 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:31.368 DEBG IO Flush 1212 has deps [JobId(1211), JobId(1210), JobId(1209)]
Sep 22 23:23:31.697 DEBG IO Write 1213 has deps [JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:31.697 DEBG up_ds_listen was notified
Sep 22 23:23:31.697 DEBG up_ds_listen process 1213
Sep 22 23:23:31.697 DEBG [A] ack job 1213:214, : downstairs
Sep 22 23:23:31.697 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:32.027 DEBG IO Write 1214 has deps [JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:32.027 DEBG up_ds_listen was notified
Sep 22 23:23:32.027 DEBG up_ds_listen process 1214
Sep 22 23:23:32.027 DEBG [A] ack job 1214:215, : downstairs
Sep 22 23:23:32.027 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:32.028 DEBG IO Flush 1215 has deps [JobId(1214), JobId(1213), JobId(1212)]
Sep 22 23:23:32.357 DEBG IO Write 1216 has deps [JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:32.357 DEBG up_ds_listen was notified
Sep 22 23:23:32.357 DEBG up_ds_listen process 1216
Sep 22 23:23:32.358 DEBG [A] ack job 1216:217, : downstairs
Sep 22 23:23:32.358 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:32.687 DEBG IO Write 1217 has deps [JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:32.687 DEBG up_ds_listen was notified
Sep 22 23:23:32.687 DEBG up_ds_listen process 1217
Sep 22 23:23:32.687 DEBG [A] ack job 1217:218, : downstairs
Sep 22 23:23:32.687 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:32.688 DEBG IO Flush 1218 has deps [JobId(1217), JobId(1216), JobId(1215)]
Sep 22 23:23:33.016 DEBG up_ds_listen process 1001
Sep 22 23:23:33.016 DEBG [A] ack job 1001:2, : downstairs
Sep 22 23:23:33.016 DEBG [rc] retire 1001 clears [JobId(1000), JobId(1001)], : downstairs
Sep 22 23:23:33.017 DEBG up_ds_listen checked 2 jobs, back to waiting
Sep 22 23:23:33.017 DEBG up_ds_listen was notified
Sep 22 23:23:33.017 DEBG up_ds_listen checked 0 jobs, back to waiting
Sep 22 23:23:33.017 DEBG IO Write 1219 has deps [JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:33.017 DEBG up_ds_listen was notified
Sep 22 23:23:33.017 DEBG up_ds_listen process 1219
Sep 22 23:23:33.017 DEBG [A] ack job 1219:220, : downstairs
Sep 22 23:23:33.017 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::integration_test_volume_replace_downstairs_then_takeover ... ok
Sep 22 23:23:33.415 DEBG IO Write 1220 has deps [JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:33.416 DEBG up_ds_listen was notified
Sep 22 23:23:33.416 DEBG up_ds_listen process 1220
Sep 22 23:23:33.416 DEBG [A] ack job 1220:221, : downstairs
Sep 22 23:23:33.416 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:33.558 DEBG IO Flush 1221 has deps [JobId(1220), JobId(1219), JobId(1218)]
Sep 22 23:23:33.558 INFO [lossy] skipping 1168
Sep 22 23:23:33.560 WARN returning error on write!
Sep 22 23:23:33.560 DEBG Write     :1168 deps:[JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:33.561 INFO [lossy] skipping 1168
Sep 22 23:23:33.591 DEBG Write     :1168 deps:[JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:33.921 DEBG IO Write 1222 has deps [JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:33.921 DEBG up_ds_listen was notified
Sep 22 23:23:33.921 DEBG up_ds_listen process 1222
Sep 22 23:23:33.921 DEBG [A] ack job 1222:223, : downstairs
Sep 22 23:23:33.921 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:34.251 DEBG IO Write 1223 has deps [JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:34.251 DEBG up_ds_listen was notified
Sep 22 23:23:34.251 DEBG up_ds_listen process 1223
Sep 22 23:23:34.251 DEBG [A] ack job 1223:224, : downstairs
Sep 22 23:23:34.251 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:34.252 DEBG IO Flush 1224 has deps [JobId(1223), JobId(1222), JobId(1221)]
Sep 22 23:23:34.581 DEBG IO Write 1225 has deps [JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:34.582 DEBG up_ds_listen was notified
Sep 22 23:23:34.582 DEBG up_ds_listen process 1225
Sep 22 23:23:34.582 DEBG [A] ack job 1225:226, : downstairs
Sep 22 23:23:34.582 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:34.912 DEBG IO Write 1226 has deps [JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:34.912 DEBG up_ds_listen was notified
Sep 22 23:23:34.912 DEBG up_ds_listen process 1226
Sep 22 23:23:34.912 DEBG [A] ack job 1226:227, : downstairs
Sep 22 23:23:34.912 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:34.913 DEBG IO Flush 1227 has deps [JobId(1226), JobId(1225), JobId(1224)]
Sep 22 23:23:35.242 DEBG IO Write 1228 has deps [JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:35.242 DEBG up_ds_listen was notified
Sep 22 23:23:35.242 DEBG up_ds_listen process 1228
Sep 22 23:23:35.242 DEBG [A] ack job 1228:229, : downstairs
Sep 22 23:23:35.242 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:35.572 DEBG IO Write 1229 has deps [JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:35.572 DEBG up_ds_listen was notified
Sep 22 23:23:35.572 DEBG up_ds_listen process 1229
Sep 22 23:23:35.572 DEBG [A] ack job 1229:230, : downstairs
Sep 22 23:23:35.572 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:35.573 DEBG IO Flush 1230 has deps [JobId(1229), JobId(1228), JobId(1227)]
Sep 22 23:23:35.902 DEBG IO Write 1231 has deps [JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:35.902 DEBG up_ds_listen was notified
Sep 22 23:23:35.902 DEBG up_ds_listen process 1231
Sep 22 23:23:35.902 DEBG [A] ack job 1231:232, : downstairs
Sep 22 23:23:35.902 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:35.903 INFO [lossy] sleeping 1 second
Sep 22 23:23:36.232 DEBG IO Write 1232 has deps [JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:23:36.233 DEBG up_ds_listen was notified
Sep 22 23:23:36.233 DEBG up_ds_listen process 1232
Sep 22 23:23:36.233 DEBG [A] ack job 1232:233, : downstairs
Sep 22 23:23:36.233 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:36.233 DEBG IO Flush 1233 has deps [JobId(1232), JobId(1231), JobId(1230)]
Sep 22 23:23:36.625 DEBG IO Write 1234 has deps [JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162), JobId(1161)]
Sep 22 23:23:36.626 DEBG up_ds_listen was notified
Sep 22 23:23:36.626 DEBG up_ds_listen process 1234
Sep 22 23:23:36.626 DEBG [A] ack job 1234:235, : downstairs
Sep 22 23:23:36.626 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:36.766 DEBG IO Read  1235 has deps [JobId(1233)]
Sep 22 23:23:36.766 DEBG IO Flush 1236 has deps [JobId(1235), JobId(1234), JobId(1233)]
Sep 22 23:23:37.002 DEBG Write     :1169 deps:[JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:37.011 DEBG Flush     :1170 extent_limit None deps:[JobId(1169), JobId(1168), JobId(1167)] res:true f:72 g:1
Sep 22 23:23:37.011 INFO [lossy] skipping 1171
Sep 22 23:23:37.011 INFO [lossy] skipping 1171
Sep 22 23:23:37.041 DEBG Write     :1171 deps:[JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:37.247 INFO [lossy] skipping 1172
Sep 22 23:23:37.278 DEBG Write     :1172 deps:[JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:37.279 INFO [lossy] skipping 1173
Sep 22 23:23:37.287 DEBG Flush     :1173 extent_limit None deps:[JobId(1172), JobId(1171), JobId(1170)] res:true f:73 g:1
Sep 22 23:23:37.491 INFO [lossy] sleeping 1 second
Sep 22 23:23:37.556 ERRO [2] job id 1163 saw error GenericError("test error")
Sep 22 23:23:37.556 ERRO [2] job id 1168 saw error GenericError("test error")
Sep 22 23:23:38.549 DEBG Write     :1174 deps:[JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.580 DEBG Write     :1175 deps:[JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.581 WARN returning error on flush!
Sep 22 23:23:38.581 DEBG Flush     :1176 extent_limit None deps:[JobId(1175), JobId(1174), JobId(1173)] res:false f:74 g:1
Sep 22 23:23:38.581 INFO [lossy] skipping 1179
Sep 22 23:23:38.581 INFO [lossy] skipping 1180
Sep 22 23:23:38.581 INFO [lossy] skipping 1181
Sep 22 23:23:38.581 INFO [lossy] skipping 1183
Sep 22 23:23:38.581 INFO [lossy] skipping 1187
Sep 22 23:23:38.581 INFO [lossy] skipping 1191
Sep 22 23:23:38.581 WARN returning error on flush!
Sep 22 23:23:38.581 DEBG Flush     :1176 extent_limit None deps:[JobId(1175), JobId(1174), JobId(1173)] res:false f:74 g:1
Sep 22 23:23:38.581 INFO [lossy] skipping 1179
Sep 22 23:23:38.581 INFO [lossy] skipping 1181
Sep 22 23:23:38.589 DEBG Flush     :1176 extent_limit None deps:[JobId(1175), JobId(1174), JobId(1173)] res:true f:74 g:1
Sep 22 23:23:38.590 WARN returning error on write!
Sep 22 23:23:38.590 DEBG Write     :1177 deps:[JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:38.590 WARN 1178 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:38.590 INFO [lossy] skipping 1180
Sep 22 23:23:38.590 WARN 1182 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:38.590 INFO [lossy] skipping 1183
Sep 22 23:23:38.590 INFO [lossy] skipping 1184
Sep 22 23:23:38.590 INFO [lossy] skipping 1185
Sep 22 23:23:38.590 WARN 1186 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:38.590 WARN 1188 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:38.590 INFO [lossy] skipping 1189
Sep 22 23:23:38.590 INFO [lossy] skipping 1190
Sep 22 23:23:38.590 WARN 1191 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:38.590 INFO [lossy] skipping 1192
Sep 22 23:23:38.590 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:38.590 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:38.591 WARN returning error on write!
Sep 22 23:23:38.591 DEBG Write     :1177 deps:[JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:38.591 WARN 1180 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:38.591 WARN 1183 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:38.591 WARN 1185 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:38.591 WARN 1189 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:38.591 INFO [lossy] skipping 1190
Sep 22 23:23:38.591 INFO [lossy] skipping 1192
Sep 22 23:23:38.621 DEBG Write     :1177 deps:[JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.626 DEBG Flush     :1178 extent_limit None deps:[JobId(1177), JobId(1176)] res:true f:75 g:1
Sep 22 23:23:38.656 DEBG Write     :1179 deps:[JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.658 WARN returning error on write!
Sep 22 23:23:38.658 DEBG Write     :1180 deps:[JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:38.658 WARN 1181 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:38.658 WARN 1182 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:38.658 INFO [lossy] skipping 1183
Sep 22 23:23:38.658 INFO [lossy] skipping 1184
Sep 22 23:23:38.658 WARN 1185 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:38.658 WARN 1186 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:38.658 INFO [lossy] skipping 1188
Sep 22 23:23:38.658 WARN 1189 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:38.658 INFO [lossy] skipping 1191
Sep 22 23:23:38.658 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:38.658 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:38.688 DEBG Write     :1180 deps:[JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.689 WARN 1183 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:38.689 WARN 1188 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:38.690 WARN 1191 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:38.697 DEBG Flush     :1181 extent_limit None deps:[JobId(1180), JobId(1179), JobId(1178)] res:true f:76 g:1
Sep 22 23:23:38.728 DEBG Write     :1182 deps:[JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.759 DEBG Write     :1183 deps:[JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.768 DEBG Flush     :1184 extent_limit None deps:[JobId(1183), JobId(1182), JobId(1181)] res:true f:77 g:1
Sep 22 23:23:38.798 DEBG Write     :1185 deps:[JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.830 DEBG Write     :1186 deps:[JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.839 DEBG Flush     :1187 extent_limit None deps:[JobId(1186), JobId(1185), JobId(1184)] res:true f:78 g:1
Sep 22 23:23:38.839 INFO [lossy] skipping 1188
Sep 22 23:23:38.869 DEBG Write     :1189 deps:[JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.870 WARN 1190 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:38.870 WARN 1191 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:38.870 INFO [lossy] skipping 1192
Sep 22 23:23:38.870 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:38.870 INFO [lossy] skipping 1194
Sep 22 23:23:38.871 WARN returning error on write!
Sep 22 23:23:38.871 DEBG Write     :1188 deps:[JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:38.872 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:38.902 DEBG Write     :1188 deps:[JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:38.903 INFO [lossy] sleeping 1 second
Sep 22 23:23:39.933 INFO [lossy] skipping 1190
Sep 22 23:23:39.933 INFO [lossy] skipping 1197
Sep 22 23:23:39.933 INFO [lossy] skipping 1204
Sep 22 23:23:39.934 INFO [lossy] skipping 1213
Sep 22 23:23:39.934 INFO [lossy] skipping 1214
Sep 22 23:23:39.934 INFO [lossy] skipping 1217
Sep 22 23:23:39.934 INFO [lossy] skipping 1218
Sep 22 23:23:39.942 DEBG Flush     :1190 extent_limit None deps:[JobId(1189), JobId(1188), JobId(1187)] res:true f:79 g:1
Sep 22 23:23:39.942 INFO [lossy] skipping 1214
Sep 22 23:23:39.942 INFO [lossy] skipping 1218
Sep 22 23:23:39.973 DEBG Write     :1191 deps:[JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:39.974 INFO [lossy] skipping 1192
Sep 22 23:23:39.974 WARN 1193 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:39.974 WARN 1194 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:39.974 WARN 1196 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:39.974 INFO [lossy] skipping 1199
Sep 22 23:23:39.974 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:39.974 INFO [lossy] skipping 1202
Sep 22 23:23:39.974 INFO [lossy] skipping 1203
Sep 22 23:23:39.974 INFO [lossy] skipping 1204
Sep 22 23:23:39.974 INFO [lossy] skipping 1205
Sep 22 23:23:39.974 WARN 1207 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:39.974 INFO [lossy] skipping 1208
Sep 22 23:23:39.974 WARN 1210 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:39.974 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:39.974 WARN 1216 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:23:39.974 INFO [lossy] skipping 1218
Sep 22 23:23:39.974 WARN returning error on flush!
Sep 22 23:23:39.974 DEBG Flush     :1192 extent_limit None deps:[JobId(1191), JobId(1190)] res:false f:80 g:1
Sep 22 23:23:39.974 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:39.974 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:39.974 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:39.974 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:39.974 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:39.978 DEBG Flush     :1192 extent_limit None deps:[JobId(1191), JobId(1190)] res:true f:80 g:1
Sep 22 23:23:39.978 INFO [lossy] sleeping 1 second
Sep 22 23:23:40.718 ERRO [2] job id 1176 saw error GenericError("test error")
Sep 22 23:23:40.718 ERRO [2] job id 1176 saw error GenericError("test error")
Sep 22 23:23:40.718 ERRO [2] job id 1177 saw error GenericError("test error")
Sep 22 23:23:40.718 ERRO [2] job id 1177 saw error GenericError("test error")
Sep 22 23:23:40.718 ERRO [2] job id 1180 saw error GenericError("test error")
Sep 22 23:23:40.718 ERRO [2] job id 1188 saw error GenericError("test error")
Sep 22 23:23:40.718 ERRO [2] job id 1192 saw error GenericError("test error")
Sep 22 23:23:41.010 DEBG Write     :1193 deps:[JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.011 INFO [lossy] skipping 1194
Sep 22 23:23:41.011 WARN 1195 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.011 WARN 1196 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.011 WARN 1197 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.011 INFO [lossy] skipping 1198
Sep 22 23:23:41.011 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:41.011 INFO [lossy] skipping 1200
Sep 22 23:23:41.011 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:41.011 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:41.011 INFO [lossy] skipping 1204
Sep 22 23:23:41.012 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:41.012 WARN 1207 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:41.012 INFO [lossy] skipping 1208
Sep 22 23:23:41.012 INFO [lossy] skipping 1209
Sep 22 23:23:41.012 INFO [lossy] skipping 1210
Sep 22 23:23:41.012 INFO [lossy] skipping 1211
Sep 22 23:23:41.012 INFO [lossy] skipping 1212
Sep 22 23:23:41.012 WARN 1213 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:41.012 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:41.012 INFO [lossy] skipping 1215
Sep 22 23:23:41.012 WARN 1216 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:41.012 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:41.012 INFO [lossy] skipping 1220
Sep 22 23:23:41.012 INFO [lossy] skipping 1226
Sep 22 23:23:41.012 INFO [lossy] skipping 1232
Sep 22 23:23:41.012 INFO [lossy] skipping 1236
Sep 22 23:23:41.042 DEBG Write     :1194 deps:[JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.043 INFO [lossy] skipping 1200
Sep 22 23:23:41.043 INFO [lossy] skipping 1208
Sep 22 23:23:41.043 INFO [lossy] skipping 1209
Sep 22 23:23:41.043 WARN 1210 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:41.043 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:41.043 INFO [lossy] skipping 1220
Sep 22 23:23:41.043 INFO [lossy] skipping 1226
Sep 22 23:23:41.043 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:41.043 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:41.051 DEBG Flush     :1195 extent_limit None deps:[JobId(1194), JobId(1193), JobId(1192)] res:true f:81 g:1
Sep 22 23:23:41.082 DEBG Write     :1196 deps:[JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.083 INFO [lossy] skipping 1197
Sep 22 23:23:41.083 INFO [lossy] skipping 1198
Sep 22 23:23:41.083 WARN 1199 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.083 INFO [lossy] skipping 1200
Sep 22 23:23:41.083 INFO [lossy] skipping 1202
Sep 22 23:23:41.083 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:41.083 INFO [lossy] skipping 1204
Sep 22 23:23:41.083 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:41.083 WARN 1207 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:41.083 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:41.083 WARN 1210 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:41.083 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:41.083 INFO [lossy] skipping 1212
Sep 22 23:23:41.083 WARN 1213 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:41.083 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:41.083 WARN 1216 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:41.083 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:41.083 INFO [lossy] skipping 1218
Sep 22 23:23:41.083 WARN 1219 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:41.083 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:41.083 WARN 1222 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:23:41.083 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:23:41.083 INFO [lossy] skipping 1224
Sep 22 23:23:41.083 WARN 1225 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:23:41.083 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:23:41.083 INFO [lossy] skipping 1227
Sep 22 23:23:41.083 INFO [lossy] skipping 1228
Sep 22 23:23:41.083 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:23:41.083 INFO [lossy] skipping 1230
Sep 22 23:23:41.083 WARN 1231 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:23:41.083 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:23:41.083 WARN 1234 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 13 deps, role: work
Sep 22 23:23:41.083 INFO [lossy] skipping 1235
Sep 22 23:23:41.083 INFO [lossy] skipping 1236
Sep 22 23:23:41.084 WARN returning error on write!
Sep 22 23:23:41.084 DEBG Write     :1197 deps:[JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:41.084 INFO [lossy] skipping 1198
Sep 22 23:23:41.084 WARN 1200 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.084 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:41.084 INFO [lossy] skipping 1204
Sep 22 23:23:41.085 INFO [lossy] skipping 1218
Sep 22 23:23:41.085 INFO [lossy] skipping 1224
Sep 22 23:23:41.085 INFO [lossy] skipping 1227
Sep 22 23:23:41.085 WARN 1228 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:23:41.115 DEBG Write     :1197 deps:[JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.124 DEBG Flush     :1198 extent_limit None deps:[JobId(1197), JobId(1196), JobId(1195)] res:true f:82 g:1
Sep 22 23:23:41.124 INFO [lossy] skipping 1218
Sep 22 23:23:41.124 INFO [lossy] skipping 1227
Sep 22 23:23:41.124 INFO [lossy] skipping 1227
Sep 22 23:23:41.125 WARN returning error on write!
Sep 22 23:23:41.125 DEBG Write     :1199 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:41.127 WARN returning error on write!
Sep 22 23:23:41.127 DEBG Write     :1200 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:41.127 INFO [lossy] skipping 1201
Sep 22 23:23:41.127 WARN 1202 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.127 INFO [lossy] skipping 1203
Sep 22 23:23:41.127 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:41.127 WARN 1207 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:41.127 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:41.127 INFO [lossy] skipping 1210
Sep 22 23:23:41.127 INFO [lossy] skipping 1211
Sep 22 23:23:41.127 INFO [lossy] skipping 1212
Sep 22 23:23:41.127 INFO [lossy] skipping 1213
Sep 22 23:23:41.127 INFO [lossy] skipping 1214
Sep 22 23:23:41.127 WARN 1216 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:41.127 INFO [lossy] skipping 1217
Sep 22 23:23:41.127 WARN 1219 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:41.127 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:41.127 INFO [lossy] skipping 1222
Sep 22 23:23:41.127 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:41.127 INFO [lossy] skipping 1224
Sep 22 23:23:41.127 INFO [lossy] skipping 1225
Sep 22 23:23:41.127 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:23:41.127 INFO [lossy] skipping 1228
Sep 22 23:23:41.127 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:23:41.127 WARN 1231 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:23:41.127 INFO [lossy] skipping 1232
Sep 22 23:23:41.127 WARN 1234 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 12 deps, role: work
Sep 22 23:23:41.127 INFO [lossy] skipping 1199
Sep 22 23:23:41.128 WARN returning error on write!
Sep 22 23:23:41.128 DEBG Write     :1200 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:41.128 INFO [lossy] skipping 1201
Sep 22 23:23:41.128 WARN 1203 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.128 WARN 1210 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:41.128 INFO [lossy] skipping 1211
Sep 22 23:23:41.128 WARN 1213 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:41.128 INFO [lossy] skipping 1214
Sep 22 23:23:41.128 INFO [lossy] skipping 1217
Sep 22 23:23:41.128 INFO [lossy] skipping 1222
Sep 22 23:23:41.128 WARN 1225 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:23:41.128 WARN 1228 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:23:41.128 INFO [lossy] skipping 1232
Sep 22 23:23:41.159 DEBG Write     :1199 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.191 DEBG Write     :1200 deps:[JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.192 INFO [lossy] skipping 1201
Sep 22 23:23:41.192 INFO [lossy] skipping 1211
Sep 22 23:23:41.192 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:41.192 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:41.192 INFO [lossy] skipping 1222
Sep 22 23:23:41.192 INFO [lossy] skipping 1232
Sep 22 23:23:41.192 INFO [lossy] skipping 1201
Sep 22 23:23:41.192 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:41.192 WARN 1222 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:41.192 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:23:41.201 DEBG Flush     :1201 extent_limit None deps:[JobId(1200), JobId(1199), JobId(1198)] res:true f:83 g:1
Sep 22 23:23:41.201 INFO [lossy] skipping 1202
Sep 22 23:23:41.231 DEBG Write     :1203 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.233 INFO [lossy] skipping 1204
Sep 22 23:23:41.233 WARN 1205 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.233 INFO [lossy] skipping 1206
Sep 22 23:23:41.233 WARN 1207 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:41.233 INFO [lossy] skipping 1208
Sep 22 23:23:41.233 INFO [lossy] skipping 1209
Sep 22 23:23:41.233 WARN 1210 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:41.233 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:41.233 INFO [lossy] skipping 1212
Sep 22 23:23:41.233 WARN 1213 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:41.233 INFO [lossy] skipping 1214
Sep 22 23:23:41.233 WARN 1216 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:41.233 INFO [lossy] skipping 1217
Sep 22 23:23:41.233 INFO [lossy] skipping 1219
Sep 22 23:23:41.233 INFO [lossy] skipping 1220
Sep 22 23:23:41.233 INFO [lossy] skipping 1221
Sep 22 23:23:41.233 INFO [lossy] skipping 1222
Sep 22 23:23:41.233 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:41.233 WARN 1225 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:41.233 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:41.233 INFO [lossy] skipping 1227
Sep 22 23:23:41.233 WARN 1228 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:23:41.233 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:23:41.233 WARN 1231 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:23:41.233 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 10 deps, role: work
Sep 22 23:23:41.233 WARN 1234 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 11 deps, role: work
Sep 22 23:23:41.233 INFO [lossy] skipping 1236
Sep 22 23:23:41.264 DEBG Write     :1202 deps:[JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.265 INFO [lossy] skipping 1204
Sep 22 23:23:41.265 WARN 1208 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:41.265 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:41.265 INFO [lossy] skipping 1217
Sep 22 23:23:41.265 WARN 1219 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:41.265 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:41.266 INFO [lossy] skipping 1221
Sep 22 23:23:41.266 INFO [lossy] skipping 1222
Sep 22 23:23:41.266 WARN returning error on flush!
Sep 22 23:23:41.266 DEBG Flush     :1204 extent_limit None deps:[JobId(1203), JobId(1202), JobId(1201)] res:false f:84 g:1
Sep 22 23:23:41.266 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:41.266 INFO [lossy] skipping 1222
Sep 22 23:23:41.266 WARN returning error on flush!
Sep 22 23:23:41.266 DEBG Flush     :1204 extent_limit None deps:[JobId(1203), JobId(1202), JobId(1201)] res:false f:84 g:1
Sep 22 23:23:41.266 WARN 1222 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:41.266 WARN returning error on flush!
Sep 22 23:23:41.266 DEBG Flush     :1204 extent_limit None deps:[JobId(1203), JobId(1202), JobId(1201)] res:false f:84 g:1
Sep 22 23:23:41.266 WARN returning error on flush!
Sep 22 23:23:41.266 DEBG Flush     :1204 extent_limit None deps:[JobId(1203), JobId(1202), JobId(1201)] res:false f:84 g:1
Sep 22 23:23:41.274 DEBG Flush     :1204 extent_limit None deps:[JobId(1203), JobId(1202), JobId(1201)] res:true f:84 g:1
Sep 22 23:23:41.305 DEBG Write     :1205 deps:[JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.310 DEBG Flush     :1206 extent_limit None deps:[JobId(1205), JobId(1204)] res:true f:85 g:1
Sep 22 23:23:41.312 WARN returning error on write!
Sep 22 23:23:41.312 DEBG Write     :1207 deps:[JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:41.343 DEBG Write     :1208 deps:[JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.344 WARN 1209 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.344 WARN 1210 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.344 WARN 1211 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:41.344 WARN 1213 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:41.344 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:41.344 WARN 1216 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:41.344 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:41.344 WARN 1219 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:41.344 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:41.344 INFO [lossy] skipping 1221
Sep 22 23:23:41.344 WARN 1222 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:41.344 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:41.345 WARN 1225 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:41.345 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:41.345 WARN 1228 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:41.345 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:41.345 WARN 1231 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:41.345 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:41.345 WARN 1234 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:23:41.345 INFO [lossy] skipping 1236
Sep 22 23:23:41.375 DEBG Write     :1207 deps:[JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:41.377 INFO [lossy] skipping 1236
Sep 22 23:23:41.377 INFO [lossy] sleeping 1 second
Sep 22 23:23:41.378 ERRO [2] job id 1197 saw error GenericError("test error")
Sep 22 23:23:41.378 ERRO [2] job id 1199 saw error GenericError("test error")
Sep 22 23:23:41.378 ERRO [2] job id 1200 saw error GenericError("test error")
Sep 22 23:23:41.378 ERRO [2] job id 1200 saw error GenericError("test error")
Sep 22 23:23:41.378 ERRO [2] job id 1204 saw error GenericError("test error")
Sep 22 23:23:41.378 ERRO [2] job id 1204 saw error GenericError("test error")
Sep 22 23:23:41.378 ERRO [2] job id 1204 saw error GenericError("test error")
Sep 22 23:23:41.378 ERRO [2] job id 1204 saw error GenericError("test error")
Sep 22 23:23:41.378 ERRO [2] job id 1207 saw error GenericError("test error")
Sep 22 23:23:42.387 DEBG Flush     :1209 extent_limit None deps:[JobId(1208), JobId(1207), JobId(1206)] res:true f:86 g:1
Sep 22 23:23:42.388 WARN returning error on write!
Sep 22 23:23:42.388 DEBG Write     :1210 deps:[JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:42.419 DEBG Write     :1211 deps:[JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:42.420 WARN 1212 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:42.420 WARN 1213 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:42.420 INFO [lossy] skipping 1214
Sep 22 23:23:42.420 INFO [lossy] skipping 1215
Sep 22 23:23:42.420 WARN 1216 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:42.420 INFO [lossy] skipping 1217
Sep 22 23:23:42.420 INFO [lossy] skipping 1219
Sep 22 23:23:42.420 INFO [lossy] skipping 1220
Sep 22 23:23:42.420 WARN 1222 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:42.420 INFO [lossy] skipping 1223
Sep 22 23:23:42.420 INFO [lossy] skipping 1225
Sep 22 23:23:42.420 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:42.420 WARN 1228 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:42.421 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:42.421 WARN 1231 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:42.421 INFO [lossy] skipping 1232
Sep 22 23:23:42.421 WARN 1234 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:23:42.451 DEBG Write     :1210 deps:[JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:42.452 WARN 1214 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:42.452 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:42.452 WARN 1219 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:42.452 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:42.452 INFO [lossy] skipping 1223
Sep 22 23:23:42.452 WARN 1225 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:42.452 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:42.452 INFO [lossy] skipping 1223
Sep 22 23:23:42.453 INFO [lossy] skipping 1223
Sep 22 23:23:42.453 INFO [lossy] skipping 1223
Sep 22 23:23:42.453 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:42.453 INFO [lossy] skipping 1212
Sep 22 23:23:42.453 INFO [lossy] skipping 1215
Sep 22 23:23:42.453 INFO [lossy] skipping 1218
Sep 22 23:23:42.453 INFO [lossy] skipping 1225
Sep 22 23:23:42.453 INFO [lossy] skipping 1227
Sep 22 23:23:42.453 INFO [lossy] skipping 1232
Sep 22 23:23:42.453 INFO [lossy] skipping 1236
Sep 22 23:23:42.461 DEBG Flush     :1212 extent_limit None deps:[JobId(1211), JobId(1210), JobId(1209)] res:true f:87 g:1
Sep 22 23:23:42.461 INFO [lossy] skipping 1215
Sep 22 23:23:42.461 WARN 1225 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:42.461 INFO [lossy] skipping 1227
Sep 22 23:23:42.461 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:42.461 INFO [lossy] skipping 1215
Sep 22 23:23:42.461 WARN 1215 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:42.461 INFO [lossy] sleeping 1 second
Sep 22 23:23:42.461 ERRO [2] job id 1210 saw error GenericError("test error")
Sep 22 23:23:43.462 INFO [lossy] skipping 1213
Sep 22 23:23:43.493 DEBG Write     :1214 deps:[JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:43.494 WARN 1215 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:43.494 WARN 1216 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:43.494 WARN 1217 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:43.494 INFO [lossy] skipping 1219
Sep 22 23:23:43.494 INFO [lossy] skipping 1220
Sep 22 23:23:43.494 WARN 1222 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:43.495 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:43.495 INFO [lossy] skipping 1224
Sep 22 23:23:43.495 INFO [lossy] skipping 1226
Sep 22 23:23:43.495 WARN 1228 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:43.495 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:43.495 INFO [lossy] skipping 1230
Sep 22 23:23:43.495 WARN 1231 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:43.495 INFO [lossy] skipping 1234
Sep 22 23:23:43.495 INFO [lossy] skipping 1235
Sep 22 23:23:43.495 INFO [lossy] skipping 1236
Sep 22 23:23:43.526 DEBG Write     :1213 deps:[JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:43.527 INFO [lossy] skipping 1219
Sep 22 23:23:43.527 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:43.527 INFO [lossy] skipping 1226
Sep 22 23:23:43.527 INFO [lossy] skipping 1230
Sep 22 23:23:43.527 WARN 1234 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:23:43.527 INFO [lossy] skipping 1235
Sep 22 23:23:43.527 WARN 1219 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:43.527 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:43.527 INFO [lossy] skipping 1235
Sep 22 23:23:43.535 DEBG Flush     :1215 extent_limit None deps:[JobId(1214), JobId(1213), JobId(1212)] res:true f:88 g:1
Sep 22 23:23:43.566 DEBG Write     :1216 deps:[JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:43.568 WARN returning error on write!
Sep 22 23:23:43.568 DEBG Write     :1217 deps:[JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:23:43.569 WARN 1218 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:43.569 INFO [lossy] skipping 1219
Sep 22 23:23:43.569 INFO [lossy] skipping 1220
Sep 22 23:23:43.569 INFO [lossy] skipping 1222
Sep 22 23:23:43.569 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:43.569 INFO [lossy] skipping 1224
Sep 22 23:23:43.569 WARN 1225 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:43.569 INFO [lossy] skipping 1226
Sep 22 23:23:43.569 WARN 1228 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:43.569 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:43.569 WARN 1231 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:43.569 INFO [lossy] skipping 1232
Sep 22 23:23:43.569 INFO [lossy] skipping 1233
Sep 22 23:23:43.569 WARN 1234 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:23:43.599 DEBG Write     :1217 deps:[JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:43.600 WARN 1219 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:43.600 WARN 1220 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:43.600 WARN 1222 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:43.600 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:43.600 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:43.600 INFO [lossy] skipping 1218
Sep 22 23:23:43.600 INFO [lossy] skipping 1222
Sep 22 23:23:43.600 INFO [lossy] skipping 1234
Sep 22 23:23:43.600 INFO [lossy] skipping 1218
Sep 22 23:23:43.600 INFO [lossy] skipping 1234
Sep 22 23:23:43.609 DEBG Flush     :1218 extent_limit None deps:[JobId(1217), JobId(1216), JobId(1215)] res:true f:89 g:1
Sep 22 23:23:43.609 WARN 1234 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:23:43.639 DEBG Write     :1219 deps:[JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:43.641 INFO [lossy] skipping 1220
Sep 22 23:23:43.641 INFO [lossy] skipping 1221
Sep 22 23:23:43.641 WARN 1222 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:43.641 WARN 1223 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:43.641 INFO [lossy] skipping 1224
Sep 22 23:23:43.641 WARN 1225 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:43.641 INFO [lossy] skipping 1226
Sep 22 23:23:43.641 WARN 1228 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:43.641 INFO [lossy] skipping 1229
Sep 22 23:23:43.641 WARN 1231 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:43.641 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:43.671 DEBG Write     :1220 deps:[JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:43.681 DEBG Flush     :1221 extent_limit None deps:[JobId(1220), JobId(1219), JobId(1218)] res:true f:90 g:1
Sep 22 23:23:43.681 WARN 1224 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:43.681 WARN 1226 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:43.681 WARN 1229 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:43.681 INFO [lossy] sleeping 1 second
Sep 22 23:23:43.682 ERRO [2] job id 1217 saw error GenericError("test error")
Sep 22 23:23:44.683 INFO [lossy] skipping 1222
Sep 22 23:23:44.713 DEBG Write     :1223 deps:[JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:44.715 INFO [lossy] skipping 1224
Sep 22 23:23:44.715 WARN 1225 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:44.715 WARN 1228 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:44.715 INFO [lossy] skipping 1231
Sep 22 23:23:44.715 INFO [lossy] skipping 1232
Sep 22 23:23:44.715 INFO [lossy] skipping 1233
Sep 22 23:23:44.715 INFO [lossy] skipping 1234
Sep 22 23:23:44.715 INFO [lossy] skipping 1235
Sep 22 23:23:44.715 INFO [lossy] skipping 1222
Sep 22 23:23:44.715 INFO [lossy] skipping 1224
Sep 22 23:23:44.715 WARN 1231 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:23:44.715 INFO [lossy] skipping 1232
Sep 22 23:23:44.715 WARN 1234 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:23:44.745 DEBG Write     :1222 deps:[JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:44.755 DEBG Flush     :1224 extent_limit None deps:[JobId(1223), JobId(1222), JobId(1221)] res:true f:91 g:1
Sep 22 23:23:44.755 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:44.785 DEBG Write     :1225 deps:[JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:44.816 DEBG Write     :1226 deps:[JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:44.826 DEBG Flush     :1227 extent_limit None deps:[JobId(1226), JobId(1225), JobId(1224)] res:true f:92 g:1
Sep 22 23:23:44.826 INFO [lossy] skipping 1228
Sep 22 23:23:44.856 DEBG Write     :1229 deps:[JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:44.857 WARN 1230 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:44.857 WARN 1231 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:44.857 WARN 1232 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:44.857 WARN 1234 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:44.888 DEBG Write     :1228 deps:[JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:44.889 WARN returning error on flush!
Sep 22 23:23:44.889 DEBG Flush     :1230 extent_limit None deps:[JobId(1229), JobId(1228), JobId(1227)] res:false f:93 g:1
Sep 22 23:23:44.889 INFO [lossy] skipping 1232
Sep 22 23:23:44.889 INFO [lossy] skipping 1236
Sep 22 23:23:44.898 DEBG Flush     :1230 extent_limit None deps:[JobId(1229), JobId(1228), JobId(1227)] res:true f:93 g:1
Sep 22 23:23:44.928 DEBG Write     :1232 deps:[JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:44.930 INFO [lossy] skipping 1236
Sep 22 23:23:44.930 INFO [lossy] skipping 1236
Sep 22 23:23:44.930 INFO [lossy] skipping 1236
Sep 22 23:23:44.930 INFO [lossy] skipping 1236
Sep 22 23:23:44.961 DEBG Write     :1231 deps:[JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:23:44.970 DEBG Flush     :1233 extent_limit None deps:[JobId(1232), JobId(1231), JobId(1230)] res:true f:94 g:1
Sep 22 23:23:44.971 WARN returning error on write!
Sep 22 23:23:44.971 DEBG Write     :1234 deps:[JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162), JobId(1161)] res:false
Sep 22 23:23:44.972 INFO [lossy] skipping 1235
Sep 22 23:23:44.972 WARN 1236 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:23:45.002 DEBG Write     :1234 deps:[JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162), JobId(1161)] res:true
Sep 22 23:23:45.010 DEBG Read      :1235 deps:[JobId(1233)] res:true
Sep 22 23:23:45.031 ERRO [2] job id 1230 saw error GenericError("test error")
Sep 22 23:23:45.031 ERRO [2] job id 1234 saw error GenericError("test error")
Sep 22 23:23:45.033 INFO [lossy] sleeping 1 second
Sep 22 23:23:45.410 DEBG [2] Read AckReady 1235, : downstairs
Sep 22 23:23:45.411 DEBG up_ds_listen was notified
Sep 22 23:23:45.411 DEBG up_ds_listen process 1235
Sep 22 23:23:45.411 DEBG [A] ack job 1235:236, : downstairs
Sep 22 23:23:45.465 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:45.466 DEBG IO Read  1237 has deps [JobId(1236)]
Sep 22 23:23:45.533 DEBG IO Flush 1238 has deps [JobId(1237), JobId(1236)]
Sep 22 23:23:46.040 DEBG Flush     :1236 extent_limit None deps:[JobId(1235), JobId(1234), JobId(1233)] res:true f:95 g:1
Sep 22 23:23:46.040 WARN returning error on read!
Sep 22 23:23:46.040 DEBG Read      :1237 deps:[JobId(1236)] res:false
Sep 22 23:23:46.040 WARN returning error on read!
Sep 22 23:23:46.040 DEBG Read      :1237 deps:[JobId(1236)] res:false
Sep 22 23:23:46.046 DEBG Read      :1237 deps:[JobId(1236)] res:true
Sep 22 23:23:46.068 ERRO [2] job id 1237 saw error GenericError("test error")
Sep 22 23:23:46.068 ERRO [2] job id 1237 saw error GenericError("test error")
Sep 22 23:23:46.069 DEBG Flush     :1238 extent_limit None deps:[JobId(1237), JobId(1236)] res:true f:96 g:1
Sep 22 23:23:46.069 INFO [lossy] sleeping 1 second
Sep 22 23:23:46.447 DEBG [2] Read AckReady 1237, : downstairs
Sep 22 23:23:46.448 DEBG up_ds_listen was notified
Sep 22 23:23:46.448 DEBG up_ds_listen process 1237
Sep 22 23:23:46.448 DEBG [A] ack job 1237:238, : downstairs
Sep 22 23:23:46.502 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:46.503 DEBG IO Read  1239 has deps [JobId(1238)]
Sep 22 23:23:46.536 DEBG IO Flush 1240 has deps [JobId(1239), JobId(1238)]
Sep 22 23:23:47.078 DEBG Read      :1239 deps:[JobId(1238)] res:true
Sep 22 23:23:47.100 INFO [lossy] skipping 1240
Sep 22 23:23:47.100 INFO [lossy] skipping 1240
Sep 22 23:23:47.100 DEBG Flush     :1240 extent_limit None deps:[JobId(1239), JobId(1238)] res:true f:97 g:1
Sep 22 23:23:47.101 INFO [lossy] sleeping 1 second
Sep 22 23:23:47.478 DEBG [2] Read AckReady 1239, : downstairs
Sep 22 23:23:47.479 DEBG up_ds_listen was notified
Sep 22 23:23:47.479 DEBG up_ds_listen process 1239
Sep 22 23:23:47.479 DEBG [A] ack job 1239:240, : downstairs
Sep 22 23:23:47.532 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:47.534 DEBG IO Read  1241 has deps [JobId(1240)]
Sep 22 23:23:47.539 DEBG IO Flush 1242 has deps [JobId(1241), JobId(1240)]
Sep 22 23:23:48.108 DEBG Read      :1241 deps:[JobId(1240)] res:true
Sep 22 23:23:48.131 INFO [lossy] skipping 1242
Sep 22 23:23:48.131 DEBG Flush     :1242 extent_limit None deps:[JobId(1241), JobId(1240)] res:true f:98 g:1
Sep 22 23:23:48.131 INFO [lossy] sleeping 1 second
Sep 22 23:23:48.509 DEBG [2] Read AckReady 1241, : downstairs
Sep 22 23:23:48.510 DEBG up_ds_listen was notified
Sep 22 23:23:48.510 DEBG up_ds_listen process 1241
Sep 22 23:23:48.510 DEBG [A] ack job 1241:242, : downstairs
Sep 22 23:23:48.562 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:48.564 DEBG IO Read  1243 has deps [JobId(1242)]
Sep 22 23:23:49.064 DEBG IO Flush 1244 has deps [JobId(1243), JobId(1242)]
Sep 22 23:23:49.140 DEBG Read      :1243 deps:[JobId(1242)] res:true
Sep 22 23:23:49.163 INFO [lossy] skipping 1244
Sep 22 23:23:49.163 DEBG Flush     :1244 extent_limit None deps:[JobId(1243), JobId(1242)] res:true f:99 g:1
Sep 22 23:23:49.163 INFO [lossy] sleeping 1 second
Sep 22 23:23:49.540 DEBG [2] Read AckReady 1243, : downstairs
Sep 22 23:23:49.541 DEBG up_ds_listen was notified
Sep 22 23:23:49.541 DEBG up_ds_listen process 1243
Sep 22 23:23:49.541 DEBG [A] ack job 1243:244, : downstairs
Sep 22 23:23:49.594 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:49.596 DEBG IO Read  1245 has deps [JobId(1244)]
Sep 22 23:23:50.096 DEBG IO Flush 1246 has deps [JobId(1245), JobId(1244)]
Sep 22 23:23:50.164 INFO [lossy] skipping 1245
Sep 22 23:23:50.170 DEBG Read      :1245 deps:[JobId(1244)] res:true
Sep 22 23:23:50.193 DEBG Flush     :1246 extent_limit None deps:[JobId(1245), JobId(1244)] res:true f:100 g:1
Sep 22 23:23:50.193 INFO [lossy] sleeping 1 second
Sep 22 23:23:50.570 DEBG [2] Read AckReady 1245, : downstairs
Sep 22 23:23:50.571 DEBG up_ds_listen was notified
Sep 22 23:23:50.571 DEBG up_ds_listen process 1245
Sep 22 23:23:50.571 DEBG [A] ack job 1245:246, : downstairs
Sep 22 23:23:50.624 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:50.625 DEBG IO Read  1247 has deps [JobId(1246)]
Sep 22 23:23:51.126 DEBG IO Flush 1248 has deps [JobId(1247), JobId(1246)]
Sep 22 23:23:51.193 INFO [lossy] skipping 1247
Sep 22 23:23:51.200 DEBG Read      :1247 deps:[JobId(1246)] res:true
Sep 22 23:23:51.223 INFO [lossy] sleeping 1 second
Sep 22 23:23:51.600 DEBG [2] Read AckReady 1247, : downstairs
Sep 22 23:23:51.601 DEBG up_ds_listen was notified
Sep 22 23:23:51.601 DEBG up_ds_listen process 1247
Sep 22 23:23:51.601 DEBG [A] ack job 1247:248, : downstairs
Sep 22 23:23:51.654 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:51.655 DEBG IO Read  1249 has deps [JobId(1248)]
Sep 22 23:23:52.156 DEBG IO Flush 1250 has deps [JobId(1249), JobId(1248)]
Sep 22 23:23:52.223 INFO [lossy] skipping 1248
Sep 22 23:23:52.223 INFO [lossy] skipping 1250
Sep 22 23:23:52.223 INFO [lossy] skipping 1248
Sep 22 23:23:52.223 INFO [lossy] skipping 1248
Sep 22 23:23:52.223 DEBG Flush     :1248 extent_limit None deps:[JobId(1247), JobId(1246)] res:true f:101 g:1
Sep 22 23:23:52.223 INFO [lossy] sleeping 1 second
Sep 22 23:23:53.225 INFO [lossy] skipping 1249
Sep 22 23:23:53.225 INFO [lossy] skipping 1250
Sep 22 23:23:53.225 INFO [lossy] skipping 1249
Sep 22 23:23:53.225 WARN 1250 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:23:53.225 WARN returning error on read!
Sep 22 23:23:53.225 DEBG Read      :1249 deps:[JobId(1248)] res:false
Sep 22 23:23:53.225 WARN returning error on read!
Sep 22 23:23:53.225 DEBG Read      :1249 deps:[JobId(1248)] res:false
Sep 22 23:23:53.225 INFO [lossy] skipping 1249
Sep 22 23:23:53.225 INFO [lossy] skipping 1249
Sep 22 23:23:53.231 DEBG Read      :1249 deps:[JobId(1248)] res:true
Sep 22 23:23:53.253 ERRO [2] job id 1249 saw error GenericError("test error")
Sep 22 23:23:53.253 ERRO [2] job id 1249 saw error GenericError("test error")
Sep 22 23:23:53.255 INFO [lossy] skipping 1250
Sep 22 23:23:53.255 WARN returning error on flush!
Sep 22 23:23:53.255 DEBG Flush     :1250 extent_limit None deps:[JobId(1249), JobId(1248)] res:false f:102 g:1
Sep 22 23:23:53.255 INFO [lossy] skipping 1250
Sep 22 23:23:53.255 WARN returning error on flush!
Sep 22 23:23:53.255 DEBG Flush     :1250 extent_limit None deps:[JobId(1249), JobId(1248)] res:false f:102 g:1
Sep 22 23:23:53.255 DEBG Flush     :1250 extent_limit None deps:[JobId(1249), JobId(1248)] res:true f:102 g:1
Sep 22 23:23:53.255 INFO [lossy] sleeping 1 second
Sep 22 23:23:53.631 DEBG [2] Read AckReady 1249, : downstairs
Sep 22 23:23:53.631 ERRO [2] job id 1250 saw error GenericError("test error")
Sep 22 23:23:53.631 ERRO [2] job id 1250 saw error GenericError("test error")
Sep 22 23:23:53.631 DEBG up_ds_listen was notified
Sep 22 23:23:53.632 DEBG up_ds_listen process 1249
Sep 22 23:23:53.632 DEBG [A] ack job 1249:250, : downstairs
Sep 22 23:23:53.684 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:53.685 DEBG IO Read  1251 has deps [JobId(1250)]
Sep 22 23:23:54.186 DEBG IO Flush 1252 has deps [JobId(1251), JobId(1250)]
Sep 22 23:23:54.255 WARN returning error on read!
Sep 22 23:23:54.255 DEBG Read      :1251 deps:[JobId(1250)] res:false
Sep 22 23:23:54.256 INFO [lossy] skipping 1251
Sep 22 23:23:54.256 WARN returning error on read!
Sep 22 23:23:54.256 DEBG Read      :1251 deps:[JobId(1250)] res:false
Sep 22 23:23:54.256 INFO [lossy] skipping 1251
Sep 22 23:23:54.262 DEBG Read      :1251 deps:[JobId(1250)] res:true
Sep 22 23:23:54.284 ERRO [2] job id 1251 saw error GenericError("test error")
Sep 22 23:23:54.284 ERRO [2] job id 1251 saw error GenericError("test error")
Sep 22 23:23:54.285 DEBG Flush     :1252 extent_limit None deps:[JobId(1251), JobId(1250)] res:true f:103 g:1
Sep 22 23:23:54.285 INFO [lossy] sleeping 1 second
Sep 22 23:23:54.661 DEBG [2] Read AckReady 1251, : downstairs
Sep 22 23:23:54.662 DEBG up_ds_listen was notified
Sep 22 23:23:54.662 DEBG up_ds_listen process 1251
Sep 22 23:23:54.662 DEBG [A] ack job 1251:252, : downstairs
Sep 22 23:23:54.715 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:54.716 DEBG IO Read  1253 has deps [JobId(1252)]
Sep 22 23:23:55.216 DEBG IO Flush 1254 has deps [JobId(1253), JobId(1252)]
Sep 22 23:23:55.293 DEBG Read      :1253 deps:[JobId(1252)] res:true
Sep 22 23:23:55.316 DEBG Flush     :1254 extent_limit None deps:[JobId(1253), JobId(1252)] res:true f:104 g:1
Sep 22 23:23:55.316 INFO [lossy] sleeping 1 second
Sep 22 23:23:55.694 DEBG [2] Read AckReady 1253, : downstairs
Sep 22 23:23:55.695 DEBG up_ds_listen was notified
Sep 22 23:23:55.695 DEBG up_ds_listen process 1253
Sep 22 23:23:55.695 DEBG [A] ack job 1253:254, : downstairs
Sep 22 23:23:55.748 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:55.749 DEBG IO Read  1255 has deps [JobId(1254)]
Sep 22 23:23:56.250 DEBG IO Flush 1256 has deps [JobId(1255), JobId(1254)]
Sep 22 23:23:56.318 WARN returning error on read!
Sep 22 23:23:56.318 DEBG Read      :1255 deps:[JobId(1254)] res:false
Sep 22 23:23:56.324 DEBG Read      :1255 deps:[JobId(1254)] res:true
Sep 22 23:23:56.345 ERRO [2] job id 1255 saw error GenericError("test error")
Sep 22 23:23:56.347 INFO [lossy] sleeping 1 second
Sep 22 23:23:56.724 DEBG [2] Read AckReady 1255, : downstairs
Sep 22 23:23:56.725 DEBG up_ds_listen was notified
Sep 22 23:23:56.725 DEBG up_ds_listen process 1255
Sep 22 23:23:56.725 DEBG [A] ack job 1255:256, : downstairs
Sep 22 23:23:56.778 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:56.779 DEBG IO Read  1257 has deps [JobId(1256)]
Sep 22 23:23:57.279 DEBG IO Flush 1258 has deps [JobId(1257), JobId(1256)]
Sep 22 23:23:57.348 DEBG Flush     :1256 extent_limit None deps:[JobId(1255), JobId(1254)] res:true f:105 g:1
Sep 22 23:23:57.355 DEBG Read      :1257 deps:[JobId(1256)] res:true
Sep 22 23:23:57.377 INFO [lossy] skipping 1258
Sep 22 23:23:57.378 DEBG Flush     :1258 extent_limit None deps:[JobId(1257), JobId(1256)] res:true f:106 g:1
Sep 22 23:23:57.378 INFO [lossy] sleeping 1 second
Sep 22 23:23:57.755 DEBG [2] Read AckReady 1257, : downstairs
Sep 22 23:23:57.756 DEBG up_ds_listen was notified
Sep 22 23:23:57.756 DEBG up_ds_listen process 1257
Sep 22 23:23:57.756 DEBG [A] ack job 1257:258, : downstairs
Sep 22 23:23:57.808 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:57.810 DEBG IO Read  1259 has deps [JobId(1258)]
Sep 22 23:23:58.311 DEBG IO Flush 1260 has deps [JobId(1259), JobId(1258)]
Sep 22 23:23:58.379 INFO [lossy] skipping 1259
Sep 22 23:23:58.379 INFO [lossy] skipping 1260
Sep 22 23:23:58.379 INFO [lossy] skipping 1259
Sep 22 23:23:58.385 DEBG Read      :1259 deps:[JobId(1258)] res:true
Sep 22 23:23:58.408 DEBG Flush     :1260 extent_limit None deps:[JobId(1259), JobId(1258)] res:true f:107 g:1
Sep 22 23:23:58.408 INFO [lossy] sleeping 1 second
Sep 22 23:23:58.786 DEBG [2] Read AckReady 1259, : downstairs
Sep 22 23:23:58.787 DEBG up_ds_listen was notified
Sep 22 23:23:58.787 DEBG up_ds_listen process 1259
Sep 22 23:23:58.787 DEBG [A] ack job 1259:260, : downstairs
Sep 22 23:23:58.840 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:58.841 DEBG IO Read  1261 has deps [JobId(1260)]
Sep 22 23:23:59.343 DEBG IO Flush 1262 has deps [JobId(1261), JobId(1260)]
Sep 22 23:23:59.410 WARN returning error on read!
Sep 22 23:23:59.410 DEBG Read      :1261 deps:[JobId(1260)] res:false
Sep 22 23:23:59.416 DEBG Read      :1261 deps:[JobId(1260)] res:true
Sep 22 23:23:59.437 ERRO [2] job id 1261 saw error GenericError("test error")
Sep 22 23:23:59.439 DEBG Flush     :1262 extent_limit None deps:[JobId(1261), JobId(1260)] res:true f:108 g:1
Sep 22 23:23:59.439 INFO [lossy] sleeping 1 second
Sep 22 23:23:59.816 DEBG [2] Read AckReady 1261, : downstairs
Sep 22 23:23:59.817 DEBG up_ds_listen was notified
Sep 22 23:23:59.817 DEBG up_ds_listen process 1261
Sep 22 23:23:59.817 DEBG [A] ack job 1261:262, : downstairs
Sep 22 23:23:59.869 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:23:59.871 DEBG IO Read  1263 has deps [JobId(1262)]
Sep 22 23:24:00.372 DEBG IO Flush 1264 has deps [JobId(1263), JobId(1262)]
Sep 22 23:24:00.446 DEBG Read      :1263 deps:[JobId(1262)] res:true
Sep 22 23:24:00.469 WARN returning error on flush!
Sep 22 23:24:00.469 DEBG Flush     :1264 extent_limit None deps:[JobId(1263), JobId(1262)] res:false f:109 g:1
Sep 22 23:24:00.469 WARN returning error on flush!
Sep 22 23:24:00.469 DEBG Flush     :1264 extent_limit None deps:[JobId(1263), JobId(1262)] res:false f:109 g:1
Sep 22 23:24:00.469 DEBG Flush     :1264 extent_limit None deps:[JobId(1263), JobId(1262)] res:true f:109 g:1
Sep 22 23:24:00.469 INFO [lossy] sleeping 1 second
Sep 22 23:24:00.845 DEBG [2] Read AckReady 1263, : downstairs
Sep 22 23:24:00.847 ERRO [2] job id 1264 saw error GenericError("test error")
Sep 22 23:24:00.847 ERRO [2] job id 1264 saw error GenericError("test error")
Sep 22 23:24:00.847 DEBG up_ds_listen was notified
Sep 22 23:24:00.847 DEBG up_ds_listen process 1263
Sep 22 23:24:00.847 DEBG [A] ack job 1263:264, : downstairs
Sep 22 23:24:00.899 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:00.901 DEBG IO Read  1265 has deps [JobId(1264)]
Sep 22 23:24:01.402 DEBG IO Flush 1266 has deps [JobId(1265), JobId(1264)]
Sep 22 23:24:01.470 WARN returning error on read!
Sep 22 23:24:01.470 DEBG Read      :1265 deps:[JobId(1264)] res:false
Sep 22 23:24:01.471 INFO [lossy] skipping 1266
Sep 22 23:24:01.471 INFO [lossy] skipping 1265
Sep 22 23:24:01.477 DEBG Read      :1265 deps:[JobId(1264)] res:true
Sep 22 23:24:01.499 ERRO [2] job id 1265 saw error GenericError("test error")
Sep 22 23:24:01.500 INFO [lossy] sleeping 1 second
Sep 22 23:24:01.877 DEBG [2] Read AckReady 1265, : downstairs
Sep 22 23:24:01.878 DEBG up_ds_listen was notified
Sep 22 23:24:01.878 DEBG up_ds_listen process 1265
Sep 22 23:24:01.878 DEBG [A] ack job 1265:266, : downstairs
Sep 22 23:24:01.930 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:01.932 DEBG IO Read  1267 has deps [JobId(1266)]
Sep 22 23:24:02.432 DEBG IO Flush 1268 has deps [JobId(1267), JobId(1266)]
Sep 22 23:24:02.502 INFO [lossy] skipping 1266
Sep 22 23:24:02.502 INFO [lossy] skipping 1267
Sep 22 23:24:02.502 DEBG Flush     :1266 extent_limit None deps:[JobId(1265), JobId(1264)] res:true f:110 g:1
Sep 22 23:24:02.508 DEBG Read      :1267 deps:[JobId(1266)] res:true
Sep 22 23:24:02.531 WARN returning error on flush!
Sep 22 23:24:02.531 DEBG Flush     :1268 extent_limit None deps:[JobId(1267), JobId(1266)] res:false f:111 g:1
Sep 22 23:24:02.531 INFO [lossy] skipping 1268
Sep 22 23:24:02.531 INFO [lossy] skipping 1268
Sep 22 23:24:02.531 DEBG Flush     :1268 extent_limit None deps:[JobId(1267), JobId(1266)] res:true f:111 g:1
Sep 22 23:24:02.531 INFO [lossy] sleeping 1 second
Sep 22 23:24:02.908 DEBG [2] Read AckReady 1267, : downstairs
Sep 22 23:24:02.909 ERRO [2] job id 1268 saw error GenericError("test error")
Sep 22 23:24:02.909 DEBG up_ds_listen was notified
Sep 22 23:24:02.909 DEBG up_ds_listen process 1267
Sep 22 23:24:02.909 DEBG [A] ack job 1267:268, : downstairs
Sep 22 23:24:02.962 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:02.963 DEBG IO Read  1269 has deps [JobId(1268)]
Sep 22 23:24:03.465 DEBG IO Flush 1270 has deps [JobId(1269), JobId(1268)]
Sep 22 23:24:03.538 DEBG Read      :1269 deps:[JobId(1268)] res:true
Sep 22 23:24:03.561 INFO [lossy] skipping 1270
Sep 22 23:24:03.561 WARN returning error on flush!
Sep 22 23:24:03.561 DEBG Flush     :1270 extent_limit None deps:[JobId(1269), JobId(1268)] res:false f:112 g:1
Sep 22 23:24:03.561 INFO [lossy] skipping 1270
Sep 22 23:24:03.561 DEBG Flush     :1270 extent_limit None deps:[JobId(1269), JobId(1268)] res:true f:112 g:1
Sep 22 23:24:03.561 INFO [lossy] sleeping 1 second
Sep 22 23:24:03.938 DEBG [2] Read AckReady 1269, : downstairs
Sep 22 23:24:03.939 ERRO [2] job id 1270 saw error GenericError("test error")
Sep 22 23:24:03.939 DEBG up_ds_listen was notified
Sep 22 23:24:03.939 DEBG up_ds_listen process 1269
Sep 22 23:24:03.939 DEBG [A] ack job 1269:270, : downstairs
Sep 22 23:24:03.991 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:03.993 DEBG IO Read  1271 has deps [JobId(1270)]
Sep 22 23:24:04.493 DEBG IO Flush 1272 has deps [JobId(1271), JobId(1270)]
Sep 22 23:24:04.563 WARN returning error on read!
Sep 22 23:24:04.563 DEBG Read      :1271 deps:[JobId(1270)] res:false
Sep 22 23:24:04.563 INFO [lossy] skipping 1271
Sep 22 23:24:04.563 WARN returning error on read!
Sep 22 23:24:04.563 DEBG Read      :1271 deps:[JobId(1270)] res:false
Sep 22 23:24:04.563 INFO [lossy] skipping 1271
Sep 22 23:24:04.563 INFO [lossy] skipping 1271
Sep 22 23:24:04.570 DEBG Read      :1271 deps:[JobId(1270)] res:true
Sep 22 23:24:04.591 ERRO [2] job id 1271 saw error GenericError("test error")
Sep 22 23:24:04.591 ERRO [2] job id 1271 saw error GenericError("test error")
Sep 22 23:24:04.593 INFO [lossy] skipping 1272
Sep 22 23:24:04.593 DEBG Flush     :1272 extent_limit None deps:[JobId(1271), JobId(1270)] res:true f:113 g:1
Sep 22 23:24:04.593 INFO [lossy] sleeping 1 second
Sep 22 23:24:04.970 DEBG [2] Read AckReady 1271, : downstairs
Sep 22 23:24:04.971 DEBG up_ds_listen was notified
Sep 22 23:24:04.971 DEBG up_ds_listen process 1271
Sep 22 23:24:04.972 DEBG [A] ack job 1271:272, : downstairs
Sep 22 23:24:05.024 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:05.025 DEBG IO Read  1273 has deps [JobId(1272)]
Sep 22 23:24:05.526 DEBG IO Flush 1274 has deps [JobId(1273), JobId(1272)]
Sep 22 23:24:05.601 DEBG Read      :1273 deps:[JobId(1272)] res:true
Sep 22 23:24:05.624 DEBG Flush     :1274 extent_limit None deps:[JobId(1273), JobId(1272)] res:true f:114 g:1
Sep 22 23:24:05.624 INFO [lossy] sleeping 1 second
Sep 22 23:24:06.002 DEBG [2] Read AckReady 1273, : downstairs
Sep 22 23:24:06.003 DEBG up_ds_listen was notified
Sep 22 23:24:06.003 DEBG up_ds_listen process 1273
Sep 22 23:24:06.003 DEBG [A] ack job 1273:274, : downstairs
Sep 22 23:24:06.056 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:06.057 DEBG IO Read  1275 has deps [JobId(1274)]
Sep 22 23:24:06.559 DEBG IO Flush 1276 has deps [JobId(1275), JobId(1274)]
Sep 22 23:24:06.625 INFO [lossy] skipping 1275
Sep 22 23:24:06.626 WARN returning error on read!
Sep 22 23:24:06.626 DEBG Read      :1275 deps:[JobId(1274)] res:false
Sep 22 23:24:06.626 WARN returning error on read!
Sep 22 23:24:06.626 DEBG Read      :1275 deps:[JobId(1274)] res:false
Sep 22 23:24:06.632 DEBG Read      :1275 deps:[JobId(1274)] res:true
Sep 22 23:24:06.654 ERRO [2] job id 1275 saw error GenericError("test error")
Sep 22 23:24:06.654 ERRO [2] job id 1275 saw error GenericError("test error")
Sep 22 23:24:06.655 DEBG Flush     :1276 extent_limit None deps:[JobId(1275), JobId(1274)] res:true f:115 g:1
Sep 22 23:24:06.655 INFO [lossy] sleeping 1 second
Sep 22 23:24:07.034 DEBG [2] Read AckReady 1275, : downstairs
Sep 22 23:24:07.036 DEBG up_ds_listen was notified
Sep 22 23:24:07.036 DEBG up_ds_listen process 1275
Sep 22 23:24:07.036 DEBG [A] ack job 1275:276, : downstairs
Sep 22 23:24:07.089 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:07.090 DEBG IO Read  1277 has deps [JobId(1276)]
Sep 22 23:24:07.590 DEBG IO Flush 1278 has deps [JobId(1277), JobId(1276)]
Sep 22 23:24:07.662 DEBG Read      :1277 deps:[JobId(1276)] res:true
Sep 22 23:24:07.685 DEBG Flush     :1278 extent_limit None deps:[JobId(1277), JobId(1276)] res:true f:116 g:1
Sep 22 23:24:07.685 INFO [lossy] sleeping 1 second
Sep 22 23:24:08.063 DEBG [2] Read AckReady 1277, : downstairs
Sep 22 23:24:08.064 DEBG up_ds_listen was notified
Sep 22 23:24:08.064 DEBG up_ds_listen process 1277
Sep 22 23:24:08.064 DEBG [A] ack job 1277:278, : downstairs
Sep 22 23:24:08.117 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:08.119 DEBG IO Read  1279 has deps [JobId(1278)]
Sep 22 23:24:08.619 DEBG IO Flush 1280 has deps [JobId(1279), JobId(1278)]
Sep 22 23:24:08.687 INFO [lossy] skipping 1279
Sep 22 23:24:08.693 DEBG Read      :1279 deps:[JobId(1278)] res:true
Sep 22 23:24:08.716 INFO [lossy] sleeping 1 second
Sep 22 23:24:09.092 DEBG [2] Read AckReady 1279, : downstairs
Sep 22 23:24:09.093 DEBG up_ds_listen was notified
Sep 22 23:24:09.093 DEBG up_ds_listen process 1279
Sep 22 23:24:09.093 DEBG [A] ack job 1279:280, : downstairs
Sep 22 23:24:09.146 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:09.147 DEBG IO Read  1281 has deps [JobId(1280)]
Sep 22 23:24:09.648 DEBG IO Flush 1282 has deps [JobId(1281), JobId(1280)]
Sep 22 23:24:09.718 INFO [lossy] skipping 1280
Sep 22 23:24:09.718 DEBG Flush     :1280 extent_limit None deps:[JobId(1279), JobId(1278)] res:true f:117 g:1
Sep 22 23:24:09.718 WARN returning error on read!
Sep 22 23:24:09.718 DEBG Read      :1281 deps:[JobId(1280)] res:false
Sep 22 23:24:09.718 WARN 1282 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:09.724 DEBG Read      :1281 deps:[JobId(1280)] res:true
Sep 22 23:24:09.746 ERRO [2] job id 1281 saw error GenericError("test error")
Sep 22 23:24:09.747 INFO [lossy] sleeping 1 second
Sep 22 23:24:10.124 DEBG [2] Read AckReady 1281, : downstairs
Sep 22 23:24:10.125 DEBG up_ds_listen was notified
Sep 22 23:24:10.126 DEBG up_ds_listen process 1281
Sep 22 23:24:10.126 DEBG [A] ack job 1281:282, : downstairs
Sep 22 23:24:10.178 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:10.180 DEBG IO Read  1283 has deps [JobId(1282)]
Sep 22 23:24:10.680 DEBG IO Flush 1284 has deps [JobId(1283), JobId(1282)]
Sep 22 23:24:10.749 DEBG Flush     :1282 extent_limit None deps:[JobId(1281), JobId(1280)] res:true f:118 g:1
Sep 22 23:24:10.755 DEBG Read      :1283 deps:[JobId(1282)] res:true
Sep 22 23:24:10.778 DEBG Flush     :1284 extent_limit None deps:[JobId(1283), JobId(1282)] res:true f:119 g:1
Sep 22 23:24:10.778 INFO [lossy] sleeping 1 second
Sep 22 23:24:11.155 DEBG [2] Read AckReady 1283, : downstairs
Sep 22 23:24:11.156 DEBG up_ds_listen was notified
Sep 22 23:24:11.156 DEBG up_ds_listen process 1283
Sep 22 23:24:11.156 DEBG [A] ack job 1283:284, : downstairs
Sep 22 23:24:11.209 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:11.211 DEBG IO Read  1285 has deps [JobId(1284)]
Sep 22 23:24:11.711 DEBG IO Flush 1286 has deps [JobId(1285), JobId(1284)]
Sep 22 23:24:11.779 WARN returning error on read!
Sep 22 23:24:11.779 DEBG Read      :1285 deps:[JobId(1284)] res:false
Sep 22 23:24:11.779 INFO [lossy] skipping 1286
Sep 22 23:24:11.785 DEBG Read      :1285 deps:[JobId(1284)] res:true
Sep 22 23:24:11.807 ERRO [2] job id 1285 saw error GenericError("test error")
Sep 22 23:24:11.808 DEBG Flush     :1286 extent_limit None deps:[JobId(1285), JobId(1284)] res:true f:120 g:1
Sep 22 23:24:12.186 DEBG [2] Read AckReady 1285, : downstairs
Sep 22 23:24:12.187 DEBG up_ds_listen was notified
Sep 22 23:24:12.187 DEBG up_ds_listen process 1285
Sep 22 23:24:12.187 DEBG [A] ack job 1285:286, : downstairs
Sep 22 23:24:12.240 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:12.241 DEBG IO Read  1287 has deps [JobId(1286)]
Sep 22 23:24:12.245 WARN returning error on read!
Sep 22 23:24:12.245 DEBG Read      :1287 deps:[JobId(1286)] res:false
Sep 22 23:24:12.251 DEBG Read      :1287 deps:[JobId(1286)] res:true
Sep 22 23:24:12.273 ERRO [2] job id 1287 saw error GenericError("test error")
Sep 22 23:24:12.650 DEBG [2] Read AckReady 1287, : downstairs
Sep 22 23:24:12.651 DEBG up_ds_listen was notified
Sep 22 23:24:12.651 DEBG up_ds_listen process 1287
Sep 22 23:24:12.651 DEBG [A] ack job 1287:288, : downstairs
Sep 22 23:24:12.704 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:12.705 DEBG IO Read  1288 has deps [JobId(1286)]
Sep 22 23:24:12.709 INFO [lossy] sleeping 1 second
Sep 22 23:24:12.742 DEBG IO Flush 1289 has deps [JobId(1288), JobId(1287), JobId(1286)]
Sep 22 23:24:13.710 INFO [lossy] skipping 1288
Sep 22 23:24:13.711 INFO [lossy] skipping 1288
Sep 22 23:24:13.717 DEBG Read      :1288 deps:[JobId(1286)] res:true
Sep 22 23:24:13.740 INFO [lossy] sleeping 1 second
Sep 22 23:24:14.116 DEBG [2] Read AckReady 1288, : downstairs
Sep 22 23:24:14.117 DEBG up_ds_listen was notified
Sep 22 23:24:14.117 DEBG up_ds_listen process 1288
Sep 22 23:24:14.117 DEBG [A] ack job 1288:289, : downstairs
Sep 22 23:24:14.170 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:14.171 DEBG IO Read  1290 has deps [JobId(1289)]
Sep 22 23:24:14.673 DEBG IO Flush 1291 has deps [JobId(1290), JobId(1289)]
Sep 22 23:24:14.741 DEBG Flush     :1289 extent_limit None deps:[JobId(1288), JobId(1287), JobId(1286)] res:true f:121 g:1
Sep 22 23:24:14.748 DEBG Read      :1290 deps:[JobId(1289)] res:true
Sep 22 23:24:14.771 WARN returning error on flush!
Sep 22 23:24:14.771 DEBG Flush     :1291 extent_limit None deps:[JobId(1290), JobId(1289)] res:false f:122 g:1
Sep 22 23:24:14.771 INFO [lossy] skipping 1291
Sep 22 23:24:14.771 WARN returning error on flush!
Sep 22 23:24:14.771 DEBG Flush     :1291 extent_limit None deps:[JobId(1290), JobId(1289)] res:false f:122 g:1
Sep 22 23:24:14.771 DEBG Flush     :1291 extent_limit None deps:[JobId(1290), JobId(1289)] res:true f:122 g:1
Sep 22 23:24:15.148 DEBG [2] Read AckReady 1290, : downstairs
Sep 22 23:24:15.149 ERRO [2] job id 1291 saw error GenericError("test error")
Sep 22 23:24:15.149 ERRO [2] job id 1291 saw error GenericError("test error")
Sep 22 23:24:15.149 DEBG up_ds_listen was notified
Sep 22 23:24:15.149 DEBG up_ds_listen process 1290
Sep 22 23:24:15.149 DEBG [A] ack job 1290:291, : downstairs
Sep 22 23:24:15.202 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:15.203 DEBG IO Read  1292 has deps [JobId(1291)]
Sep 22 23:24:15.213 DEBG Read      :1292 deps:[JobId(1291)] res:true
Sep 22 23:24:15.612 DEBG [2] Read AckReady 1292, : downstairs
Sep 22 23:24:15.613 DEBG up_ds_listen was notified
Sep 22 23:24:15.613 DEBG up_ds_listen process 1292
Sep 22 23:24:15.613 DEBG [A] ack job 1292:293, : downstairs
Sep 22 23:24:15.665 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:15.666 DEBG IO Read  1293 has deps [JobId(1291)]
Sep 22 23:24:15.671 INFO [lossy] sleeping 1 second
Sep 22 23:24:15.704 DEBG IO Flush 1294 has deps [JobId(1293), JobId(1292), JobId(1291)]
Sep 22 23:24:16.678 DEBG Read      :1293 deps:[JobId(1291)] res:true
Sep 22 23:24:16.701 DEBG Flush     :1294 extent_limit None deps:[JobId(1293), JobId(1292), JobId(1291)] res:true f:123 g:1
Sep 22 23:24:16.701 INFO [lossy] sleeping 1 second
Sep 22 23:24:17.078 DEBG [2] Read AckReady 1293, : downstairs
Sep 22 23:24:17.079 DEBG up_ds_listen was notified
Sep 22 23:24:17.079 DEBG up_ds_listen process 1293
Sep 22 23:24:17.079 DEBG [A] ack job 1293:294, : downstairs
Sep 22 23:24:17.131 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:17.133 DEBG IO Read  1295 has deps [JobId(1294)]
Sep 22 23:24:17.633 DEBG IO Flush 1296 has deps [JobId(1295), JobId(1294)]
Sep 22 23:24:17.709 DEBG Read      :1295 deps:[JobId(1294)] res:true
Sep 22 23:24:17.731 WARN returning error on flush!
Sep 22 23:24:17.731 DEBG Flush     :1296 extent_limit None deps:[JobId(1295), JobId(1294)] res:false f:124 g:1
Sep 22 23:24:17.731 INFO [lossy] skipping 1296
Sep 22 23:24:17.731 DEBG Flush     :1296 extent_limit None deps:[JobId(1295), JobId(1294)] res:true f:124 g:1
Sep 22 23:24:18.107 DEBG [2] Read AckReady 1295, : downstairs
Sep 22 23:24:18.108 ERRO [2] job id 1296 saw error GenericError("test error")
Sep 22 23:24:18.108 DEBG up_ds_listen was notified
Sep 22 23:24:18.108 DEBG up_ds_listen process 1295
Sep 22 23:24:18.109 DEBG [A] ack job 1295:296, : downstairs
Sep 22 23:24:18.161 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:18.162 DEBG IO Read  1297 has deps [JobId(1296)]
Sep 22 23:24:18.166 INFO [lossy] sleeping 1 second
Sep 22 23:24:18.664 DEBG IO Flush 1298 has deps [JobId(1297), JobId(1296)]
Sep 22 23:24:19.174 DEBG Read      :1297 deps:[JobId(1296)] res:true
Sep 22 23:24:19.197 DEBG Flush     :1298 extent_limit None deps:[JobId(1297), JobId(1296)] res:true f:125 g:1
Sep 22 23:24:19.574 DEBG [2] Read AckReady 1297, : downstairs
Sep 22 23:24:19.575 DEBG up_ds_listen was notified
Sep 22 23:24:19.575 DEBG up_ds_listen process 1297
Sep 22 23:24:19.575 DEBG [A] ack job 1297:298, : downstairs
Sep 22 23:24:19.628 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:19.630 DEBG IO Read  1299 has deps [JobId(1298)]
Sep 22 23:24:19.640 DEBG Read      :1299 deps:[JobId(1298)] res:true
Sep 22 23:24:20.039 DEBG [2] Read AckReady 1299, : downstairs
Sep 22 23:24:20.040 DEBG up_ds_listen was notified
Sep 22 23:24:20.040 DEBG up_ds_listen process 1299
Sep 22 23:24:20.040 DEBG [A] ack job 1299:300, : downstairs
Sep 22 23:24:20.093 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:20.094 DEBG IO Flush 1300 has deps [JobId(1299), JobId(1298)]
Sep 22 23:24:20.094 DEBG IO Read  1301 has deps [JobId(1300)]
Sep 22 23:24:20.096 INFO [lossy] skipping 1300
Sep 22 23:24:20.096 INFO [lossy] skipping 1300
Sep 22 23:24:20.097 DEBG Flush     :1300 extent_limit None deps:[JobId(1299), JobId(1298)] res:true f:126 g:1
Sep 22 23:24:20.105 DEBG Read      :1301 deps:[JobId(1300)] res:true
Sep 22 23:24:20.503 DEBG [2] Read AckReady 1301, : downstairs
Sep 22 23:24:20.504 DEBG up_ds_listen was notified
Sep 22 23:24:20.504 DEBG up_ds_listen process 1301
Sep 22 23:24:20.504 DEBG [A] ack job 1301:302, : downstairs
Sep 22 23:24:20.557 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:20.558 DEBG IO Read  1302 has deps [JobId(1300)]
Sep 22 23:24:20.568 DEBG Read      :1302 deps:[JobId(1300)] res:true
Sep 22 23:24:20.967 DEBG [2] Read AckReady 1302, : downstairs
Sep 22 23:24:20.968 DEBG up_ds_listen was notified
Sep 22 23:24:20.968 DEBG up_ds_listen process 1302
Sep 22 23:24:20.968 DEBG [A] ack job 1302:303, : downstairs
Sep 22 23:24:21.021 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:21.022 DEBG IO Flush 1303 has deps [JobId(1302), JobId(1301), JobId(1300)]
Sep 22 23:24:21.022 DEBG IO Read  1304 has deps [JobId(1303)]
Sep 22 23:24:21.024 INFO [lossy] skipping 1303
Sep 22 23:24:21.024 DEBG Flush     :1303 extent_limit None deps:[JobId(1302), JobId(1301), JobId(1300)] res:true f:127 g:1
Sep 22 23:24:21.027 WARN returning error on read!
Sep 22 23:24:21.027 DEBG Read      :1304 deps:[JobId(1303)] res:false
Sep 22 23:24:21.033 DEBG Read      :1304 deps:[JobId(1303)] res:true
Sep 22 23:24:21.054 ERRO [2] job id 1304 saw error GenericError("test error")
Sep 22 23:24:21.433 DEBG [2] Read AckReady 1304, : downstairs
Sep 22 23:24:21.434 DEBG up_ds_listen was notified
Sep 22 23:24:21.434 DEBG up_ds_listen process 1304
Sep 22 23:24:21.435 DEBG [A] ack job 1304:305, : downstairs
Sep 22 23:24:21.487 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:21.488 DEBG IO Read  1305 has deps [JobId(1303)]
Sep 22 23:24:21.498 DEBG Read      :1305 deps:[JobId(1303)] res:true
Sep 22 23:24:21.897 DEBG [2] Read AckReady 1305, : downstairs
Sep 22 23:24:21.898 DEBG up_ds_listen was notified
Sep 22 23:24:21.898 DEBG up_ds_listen process 1305
Sep 22 23:24:21.898 DEBG [A] ack job 1305:306, : downstairs
Sep 22 23:24:21.951 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:21.952 DEBG IO Flush 1306 has deps [JobId(1305), JobId(1304), JobId(1303)]
Sep 22 23:24:21.952 DEBG IO Read  1307 has deps [JobId(1306)]
Sep 22 23:24:21.954 INFO [lossy] sleeping 1 second
Sep 22 23:24:22.453 DEBG IO Flush 1308 has deps [JobId(1307), JobId(1306)]
Sep 22 23:24:22.955 WARN returning error on flush!
Sep 22 23:24:22.955 DEBG Flush     :1306 extent_limit None deps:[JobId(1305), JobId(1304), JobId(1303)] res:false f:128 g:1
Sep 22 23:24:22.955 WARN returning error on flush!
Sep 22 23:24:22.955 DEBG Flush     :1306 extent_limit None deps:[JobId(1305), JobId(1304), JobId(1303)] res:false f:128 g:1
Sep 22 23:24:22.955 DEBG Flush     :1306 extent_limit None deps:[JobId(1305), JobId(1304), JobId(1303)] res:true f:128 g:1
Sep 22 23:24:22.955 INFO [lossy] skipping 1307
Sep 22 23:24:22.955 WARN 1308 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:22.961 DEBG Read      :1307 deps:[JobId(1306)] res:true
Sep 22 23:24:22.983 ERRO [2] job id 1306 saw error GenericError("test error")
Sep 22 23:24:22.983 ERRO [2] job id 1306 saw error GenericError("test error")
Sep 22 23:24:22.985 WARN returning error on flush!
Sep 22 23:24:22.985 DEBG Flush     :1308 extent_limit None deps:[JobId(1307), JobId(1306)] res:false f:129 g:1
Sep 22 23:24:22.985 DEBG Flush     :1308 extent_limit None deps:[JobId(1307), JobId(1306)] res:true f:129 g:1
Sep 22 23:24:23.362 DEBG [2] Read AckReady 1307, : downstairs
Sep 22 23:24:23.363 ERRO [2] job id 1308 saw error GenericError("test error")
Sep 22 23:24:23.363 DEBG up_ds_listen was notified
Sep 22 23:24:23.363 DEBG up_ds_listen process 1307
Sep 22 23:24:23.363 DEBG [A] ack job 1307:308, : downstairs
Sep 22 23:24:23.416 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:23.417 DEBG IO Read  1309 has deps [JobId(1308)]
Sep 22 23:24:23.421 WARN returning error on read!
Sep 22 23:24:23.421 DEBG Read      :1309 deps:[JobId(1308)] res:false
Sep 22 23:24:23.421 WARN returning error on read!
Sep 22 23:24:23.421 DEBG Read      :1309 deps:[JobId(1308)] res:false
Sep 22 23:24:23.421 INFO [lossy] skipping 1309
Sep 22 23:24:23.427 DEBG Read      :1309 deps:[JobId(1308)] res:true
Sep 22 23:24:23.448 ERRO [2] job id 1309 saw error GenericError("test error")
Sep 22 23:24:23.449 ERRO [2] job id 1309 saw error GenericError("test error")
Sep 22 23:24:23.825 DEBG [2] Read AckReady 1309, : downstairs
Sep 22 23:24:23.827 DEBG up_ds_listen was notified
Sep 22 23:24:23.827 DEBG up_ds_listen process 1309
Sep 22 23:24:23.827 DEBG [A] ack job 1309:310, : downstairs
Sep 22 23:24:23.879 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:23.880 DEBG IO Flush 1310 has deps [JobId(1309), JobId(1308)]
Sep 22 23:24:23.881 DEBG IO Read  1311 has deps [JobId(1310)]
Sep 22 23:24:23.883 INFO [lossy] sleeping 1 second
Sep 22 23:24:24.381 DEBG IO Flush 1312 has deps [JobId(1311), JobId(1310)]
Sep 22 23:24:24.884 DEBG Flush     :1310 extent_limit None deps:[JobId(1309), JobId(1308)] res:true f:130 g:1
Sep 22 23:24:24.891 DEBG Read      :1311 deps:[JobId(1310)] res:true
Sep 22 23:24:24.914 DEBG Flush     :1312 extent_limit None deps:[JobId(1311), JobId(1310)] res:true f:131 g:1
Sep 22 23:24:25.291 DEBG [2] Read AckReady 1311, : downstairs
Sep 22 23:24:25.292 DEBG up_ds_listen was notified
Sep 22 23:24:25.292 DEBG up_ds_listen process 1311
Sep 22 23:24:25.292 DEBG [A] ack job 1311:312, : downstairs
Sep 22 23:24:25.345 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:25.347 DEBG IO Read  1313 has deps [JobId(1312)]
Sep 22 23:24:25.351 WARN returning error on read!
Sep 22 23:24:25.351 DEBG Read      :1313 deps:[JobId(1312)] res:false
Sep 22 23:24:25.357 DEBG Read      :1313 deps:[JobId(1312)] res:true
Sep 22 23:24:25.379 ERRO [2] job id 1313 saw error GenericError("test error")
Sep 22 23:24:25.759 DEBG [2] Read AckReady 1313, : downstairs
Sep 22 23:24:25.760 DEBG up_ds_listen was notified
Sep 22 23:24:25.760 DEBG up_ds_listen process 1313
Sep 22 23:24:25.760 DEBG [A] ack job 1313:314, : downstairs
Sep 22 23:24:25.812 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:25.813 DEBG IO Flush 1314 has deps [JobId(1313), JobId(1312)]
Sep 22 23:24:25.814 DEBG IO Read  1315 has deps [JobId(1314)]
Sep 22 23:24:25.816 DEBG Flush     :1314 extent_limit None deps:[JobId(1313), JobId(1312)] res:true f:132 g:1
Sep 22 23:24:25.818 INFO [lossy] sleeping 1 second
Sep 22 23:24:26.315 DEBG IO Flush 1316 has deps [JobId(1315), JobId(1314)]
Sep 22 23:24:26.825 DEBG Read      :1315 deps:[JobId(1314)] res:true
Sep 22 23:24:26.848 WARN returning error on flush!
Sep 22 23:24:26.848 DEBG Flush     :1316 extent_limit None deps:[JobId(1315), JobId(1314)] res:false f:133 g:1
Sep 22 23:24:26.848 INFO [lossy] skipping 1316
Sep 22 23:24:26.848 DEBG Flush     :1316 extent_limit None deps:[JobId(1315), JobId(1314)] res:true f:133 g:1
Sep 22 23:24:27.225 DEBG [2] Read AckReady 1315, : downstairs
Sep 22 23:24:27.226 ERRO [2] job id 1316 saw error GenericError("test error")
Sep 22 23:24:27.226 DEBG up_ds_listen was notified
Sep 22 23:24:27.226 DEBG up_ds_listen process 1315
Sep 22 23:24:27.226 DEBG [A] ack job 1315:316, : downstairs
Sep 22 23:24:27.279 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:27.280 DEBG IO Read  1317 has deps [JobId(1316)]
Sep 22 23:24:27.290 DEBG Read      :1317 deps:[JobId(1316)] res:true
Sep 22 23:24:27.689 DEBG [2] Read AckReady 1317, : downstairs
Sep 22 23:24:27.691 DEBG up_ds_listen was notified
Sep 22 23:24:27.691 DEBG up_ds_listen process 1317
Sep 22 23:24:27.691 DEBG [A] ack job 1317:318, : downstairs
Sep 22 23:24:27.743 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:27.744 DEBG IO Flush 1318 has deps [JobId(1317), JobId(1316)]
Sep 22 23:24:27.745 DEBG IO Read  1319 has deps [JobId(1318)]
Sep 22 23:24:27.747 WARN returning error on flush!
Sep 22 23:24:27.747 DEBG Flush     :1318 extent_limit None deps:[JobId(1317), JobId(1316)] res:false f:134 g:1
Sep 22 23:24:27.747 WARN returning error on flush!
Sep 22 23:24:27.747 DEBG Flush     :1318 extent_limit None deps:[JobId(1317), JobId(1316)] res:false f:134 g:1
Sep 22 23:24:27.747 DEBG Flush     :1318 extent_limit None deps:[JobId(1317), JobId(1316)] res:true f:134 g:1
Sep 22 23:24:27.749 ERRO [2] job id 1318 saw error GenericError("test error")
Sep 22 23:24:27.749 ERRO [2] job id 1318 saw error GenericError("test error")
Sep 22 23:24:27.749 WARN returning error on read!
Sep 22 23:24:27.749 DEBG Read      :1319 deps:[JobId(1318)] res:false
Sep 22 23:24:27.750 WARN returning error on read!
Sep 22 23:24:27.750 DEBG Read      :1319 deps:[JobId(1318)] res:false
Sep 22 23:24:27.755 DEBG Read      :1319 deps:[JobId(1318)] res:true
Sep 22 23:24:27.777 ERRO [2] job id 1319 saw error GenericError("test error")
Sep 22 23:24:27.777 ERRO [2] job id 1319 saw error GenericError("test error")
Sep 22 23:24:28.155 DEBG [2] Read AckReady 1319, : downstairs
Sep 22 23:24:28.156 DEBG up_ds_listen was notified
Sep 22 23:24:28.156 DEBG up_ds_listen process 1319
Sep 22 23:24:28.156 DEBG [A] ack job 1319:320, : downstairs
Sep 22 23:24:28.209 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:28.210 DEBG IO Read  1320 has deps [JobId(1318)]
Sep 22 23:24:28.214 WARN returning error on read!
Sep 22 23:24:28.214 DEBG Read      :1320 deps:[JobId(1318)] res:false
Sep 22 23:24:28.220 DEBG Read      :1320 deps:[JobId(1318)] res:true
Sep 22 23:24:28.241 ERRO [2] job id 1320 saw error GenericError("test error")
Sep 22 23:24:28.619 DEBG [2] Read AckReady 1320, : downstairs
Sep 22 23:24:28.620 DEBG up_ds_listen was notified
Sep 22 23:24:28.620 DEBG up_ds_listen process 1320
Sep 22 23:24:28.620 DEBG [A] ack job 1320:321, : downstairs
Sep 22 23:24:28.673 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:28.674 DEBG IO Flush 1321 has deps [JobId(1320), JobId(1319), JobId(1318)]
Sep 22 23:24:28.674 DEBG IO Read  1322 has deps [JobId(1321)]
Sep 22 23:24:28.676 INFO [lossy] skipping 1321
Sep 22 23:24:28.676 DEBG Flush     :1321 extent_limit None deps:[JobId(1320), JobId(1319), JobId(1318)] res:true f:135 g:1
Sep 22 23:24:28.679 WARN returning error on read!
Sep 22 23:24:28.679 DEBG Read      :1322 deps:[JobId(1321)] res:false
Sep 22 23:24:28.685 DEBG Read      :1322 deps:[JobId(1321)] res:true
Sep 22 23:24:28.706 ERRO [2] job id 1322 saw error GenericError("test error")
Sep 22 23:24:29.084 DEBG [2] Read AckReady 1322, : downstairs
Sep 22 23:24:29.085 DEBG up_ds_listen was notified
Sep 22 23:24:29.085 DEBG up_ds_listen process 1322
Sep 22 23:24:29.085 DEBG [A] ack job 1322:323, : downstairs
Sep 22 23:24:29.138 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:31.253 DEBG IO Write 1323 has deps [JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1235), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1163), JobId(1162)]
Sep 22 23:24:31.253 DEBG IO Flush 1324 has deps [JobId(1323), JobId(1322), JobId(1321)]
Sep 22 23:24:31.253 DEBG up_ds_listen was notified
Sep 22 23:24:31.253 DEBG up_ds_listen process 1323
Sep 22 23:24:31.254 DEBG [A] ack job 1323:324, : downstairs
Sep 22 23:24:31.254 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:31.604 DEBG IO Write 1325 has deps [JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1237), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1165), JobId(1164), JobId(1162)]
Sep 22 23:24:31.604 DEBG up_ds_listen was notified
Sep 22 23:24:31.604 DEBG up_ds_listen process 1325
Sep 22 23:24:31.605 DEBG [A] ack job 1325:326, : downstairs
Sep 22 23:24:31.605 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:31.605 INFO [lossy] sleeping 1 second
Sep 22 23:24:31.935 DEBG IO Write 1326 has deps [JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1239), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1166), JobId(1164), JobId(1162)]
Sep 22 23:24:31.935 DEBG up_ds_listen was notified
Sep 22 23:24:31.935 DEBG up_ds_listen process 1326
Sep 22 23:24:31.935 DEBG [A] ack job 1326:327, : downstairs
Sep 22 23:24:31.935 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:31.935 DEBG IO Flush 1327 has deps [JobId(1326), JobId(1325), JobId(1324)]
Sep 22 23:24:32.265 DEBG IO Write 1328 has deps [JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1241), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1168), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:32.265 DEBG up_ds_listen was notified
Sep 22 23:24:32.265 DEBG up_ds_listen process 1328
Sep 22 23:24:32.265 DEBG [A] ack job 1328:329, : downstairs
Sep 22 23:24:32.265 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:32.595 DEBG IO Write 1329 has deps [JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1243), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1169), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:32.595 DEBG up_ds_listen was notified
Sep 22 23:24:32.595 DEBG up_ds_listen process 1329
Sep 22 23:24:32.595 DEBG [A] ack job 1329:330, : downstairs
Sep 22 23:24:32.595 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:32.596 DEBG IO Flush 1330 has deps [JobId(1329), JobId(1328), JobId(1327)]
Sep 22 23:24:32.925 DEBG IO Write 1331 has deps [JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1245), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1171), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:32.925 DEBG up_ds_listen was notified
Sep 22 23:24:32.926 DEBG up_ds_listen process 1331
Sep 22 23:24:32.926 DEBG [A] ack job 1331:332, : downstairs
Sep 22 23:24:32.926 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:33.255 DEBG IO Write 1332 has deps [JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1247), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1172), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:33.255 DEBG up_ds_listen was notified
Sep 22 23:24:33.255 DEBG up_ds_listen process 1332
Sep 22 23:24:33.256 DEBG [A] ack job 1332:333, : downstairs
Sep 22 23:24:33.256 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:33.256 DEBG IO Flush 1333 has deps [JobId(1332), JobId(1331), JobId(1330)]
Sep 22 23:24:33.586 DEBG IO Write 1334 has deps [JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1249), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1174), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:33.586 DEBG up_ds_listen was notified
Sep 22 23:24:33.586 DEBG up_ds_listen process 1334
Sep 22 23:24:33.586 DEBG [A] ack job 1334:335, : downstairs
Sep 22 23:24:33.586 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:33.916 DEBG IO Write 1335 has deps [JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1251), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1175), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:33.916 DEBG up_ds_listen was notified
Sep 22 23:24:33.916 DEBG up_ds_listen process 1335
Sep 22 23:24:33.916 DEBG [A] ack job 1335:336, : downstairs
Sep 22 23:24:33.916 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:33.917 DEBG IO Flush 1336 has deps [JobId(1335), JobId(1334), JobId(1333)]
Sep 22 23:24:34.267 DEBG IO Write 1337 has deps [JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1253), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1177), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:34.267 DEBG up_ds_listen was notified
Sep 22 23:24:34.267 DEBG up_ds_listen process 1337
Sep 22 23:24:34.267 DEBG [A] ack job 1337:338, : downstairs
Sep 22 23:24:34.267 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:34.314 INFO [lossy] skipping 1323
Sep 22 23:24:34.314 INFO [lossy] skipping 1324
Sep 22 23:24:34.315 WARN returning error on write!
Sep 22 23:24:34.315 DEBG Write     :1323 deps:[JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1235), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1163), JobId(1162)] res:false
Sep 22 23:24:34.316 WARN returning error on write!
Sep 22 23:24:34.316 DEBG Write     :1323 deps:[JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1235), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1163), JobId(1162)] res:false
Sep 22 23:24:34.317 WARN returning error on write!
Sep 22 23:24:34.317 DEBG Write     :1323 deps:[JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1235), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1163), JobId(1162)] res:false
Sep 22 23:24:34.347 DEBG Write     :1323 deps:[JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1235), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1163), JobId(1162)] res:true
Sep 22 23:24:34.352 DEBG Flush     :1324 extent_limit None deps:[JobId(1323), JobId(1322), JobId(1321)] res:true f:136 g:1
Sep 22 23:24:34.681 DEBG IO Write 1338 has deps [JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1255), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1179), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:34.681 DEBG up_ds_listen was notified
Sep 22 23:24:34.681 DEBG up_ds_listen process 1338
Sep 22 23:24:34.681 DEBG [A] ack job 1338:339, : downstairs
Sep 22 23:24:34.681 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:34.682 DEBG IO Flush 1339 has deps [JobId(1338), JobId(1337), JobId(1336)]
Sep 22 23:24:35.011 DEBG IO Write 1340 has deps [JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1257), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1180), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:35.011 DEBG up_ds_listen was notified
Sep 22 23:24:35.011 DEBG up_ds_listen process 1340
Sep 22 23:24:35.011 DEBG [A] ack job 1340:341, : downstairs
Sep 22 23:24:35.011 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:35.341 DEBG IO Write 1341 has deps [JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1259), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1182), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:35.341 DEBG up_ds_listen was notified
Sep 22 23:24:35.341 DEBG up_ds_listen process 1341
Sep 22 23:24:35.341 DEBG [A] ack job 1341:342, : downstairs
Sep 22 23:24:35.341 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:35.342 DEBG IO Flush 1342 has deps [JobId(1341), JobId(1340), JobId(1339)]
Sep 22 23:24:35.671 DEBG IO Write 1343 has deps [JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1261), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1183), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:35.671 DEBG up_ds_listen was notified
Sep 22 23:24:35.671 DEBG up_ds_listen process 1343
Sep 22 23:24:35.672 DEBG [A] ack job 1343:344, : downstairs
Sep 22 23:24:35.672 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:36.002 DEBG IO Write 1344 has deps [JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1263), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1185), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:36.002 DEBG up_ds_listen was notified
Sep 22 23:24:36.002 DEBG up_ds_listen process 1344
Sep 22 23:24:36.002 DEBG [A] ack job 1344:345, : downstairs
Sep 22 23:24:36.002 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:36.003 DEBG IO Flush 1345 has deps [JobId(1344), JobId(1343), JobId(1342)]
Sep 22 23:24:36.332 DEBG IO Write 1346 has deps [JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1265), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1186), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:36.332 DEBG up_ds_listen was notified
Sep 22 23:24:36.332 DEBG up_ds_listen process 1346
Sep 22 23:24:36.332 DEBG [A] ack job 1346:347, : downstairs
Sep 22 23:24:36.332 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:36.662 DEBG IO Write 1347 has deps [JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1267), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1188), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:36.662 DEBG up_ds_listen was notified
Sep 22 23:24:36.662 DEBG up_ds_listen process 1347
Sep 22 23:24:36.662 DEBG [A] ack job 1347:348, : downstairs
Sep 22 23:24:36.662 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:36.662 DEBG IO Flush 1348 has deps [JobId(1347), JobId(1346), JobId(1345)]
Sep 22 23:24:36.992 DEBG IO Write 1349 has deps [JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1269), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1189), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:36.992 DEBG up_ds_listen was notified
Sep 22 23:24:36.992 DEBG up_ds_listen process 1349
Sep 22 23:24:36.992 DEBG [A] ack job 1349:350, : downstairs
Sep 22 23:24:36.992 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:37.343 DEBG IO Write 1350 has deps [JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1271), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1191), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:37.343 DEBG up_ds_listen was notified
Sep 22 23:24:37.343 DEBG up_ds_listen process 1350
Sep 22 23:24:37.343 DEBG [A] ack job 1350:351, : downstairs
Sep 22 23:24:37.343 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:37.390 DEBG IO Flush 1351 has deps [JobId(1350), JobId(1349), JobId(1348)]
Sep 22 23:24:37.390 INFO [lossy] skipping 1325
Sep 22 23:24:37.390 INFO [lossy] skipping 1325
Sep 22 23:24:37.420 DEBG Write     :1325 deps:[JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1237), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1165), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:37.750 DEBG IO Write 1352 has deps [JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1273), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1193), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:37.750 DEBG up_ds_listen was notified
Sep 22 23:24:37.750 DEBG up_ds_listen process 1352
Sep 22 23:24:37.750 DEBG [A] ack job 1352:353, : downstairs
Sep 22 23:24:37.750 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:38.079 DEBG IO Write 1353 has deps [JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1275), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1194), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:38.080 DEBG up_ds_listen was notified
Sep 22 23:24:38.080 DEBG up_ds_listen process 1353
Sep 22 23:24:38.080 DEBG [A] ack job 1353:354, : downstairs
Sep 22 23:24:38.080 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:38.080 DEBG IO Flush 1354 has deps [JobId(1353), JobId(1352), JobId(1351)]
Sep 22 23:24:38.409 DEBG IO Write 1355 has deps [JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1277), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1196), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:38.410 DEBG up_ds_listen was notified
Sep 22 23:24:38.410 DEBG up_ds_listen process 1355
Sep 22 23:24:38.410 DEBG [A] ack job 1355:356, : downstairs
Sep 22 23:24:38.410 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:38.740 DEBG IO Write 1356 has deps [JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1279), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1197), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:38.740 DEBG up_ds_listen was notified
Sep 22 23:24:38.740 DEBG up_ds_listen process 1356
Sep 22 23:24:38.740 DEBG [A] ack job 1356:357, : downstairs
Sep 22 23:24:38.740 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:38.741 DEBG IO Flush 1357 has deps [JobId(1356), JobId(1355), JobId(1354)]
Sep 22 23:24:39.070 DEBG IO Write 1358 has deps [JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1281), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1199), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:39.070 DEBG up_ds_listen was notified
Sep 22 23:24:39.070 DEBG up_ds_listen process 1358
Sep 22 23:24:39.070 DEBG [A] ack job 1358:359, : downstairs
Sep 22 23:24:39.070 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:39.399 DEBG IO Write 1359 has deps [JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1283), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1200), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:39.400 DEBG up_ds_listen was notified
Sep 22 23:24:39.400 DEBG up_ds_listen process 1359
Sep 22 23:24:39.400 DEBG [A] ack job 1359:360, : downstairs
Sep 22 23:24:39.400 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:39.400 DEBG IO Flush 1360 has deps [JobId(1359), JobId(1358), JobId(1357)]
Sep 22 23:24:39.729 DEBG IO Write 1361 has deps [JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1285), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1202), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:39.730 DEBG up_ds_listen was notified
Sep 22 23:24:39.730 DEBG up_ds_listen process 1361
Sep 22 23:24:39.730 DEBG [A] ack job 1361:362, : downstairs
Sep 22 23:24:39.730 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:40.060 DEBG IO Write 1362 has deps [JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1287), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1203), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:40.060 DEBG up_ds_listen was notified
Sep 22 23:24:40.060 DEBG up_ds_listen process 1362
Sep 22 23:24:40.060 DEBG [A] ack job 1362:363, : downstairs
Sep 22 23:24:40.060 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:40.061 DEBG IO Flush 1363 has deps [JobId(1362), JobId(1361), JobId(1360)]
Sep 22 23:24:40.412 DEBG IO Write 1364 has deps [JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1288), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1205), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:40.413 DEBG up_ds_listen was notified
Sep 22 23:24:40.413 DEBG up_ds_listen process 1364
Sep 22 23:24:40.413 DEBG [A] ack job 1364:365, : downstairs
Sep 22 23:24:40.413 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:40.490 DEBG Write     :1326 deps:[JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1239), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1166), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:40.491 INFO [lossy] skipping 1327
Sep 22 23:24:40.491 INFO [lossy] skipping 1327
Sep 22 23:24:40.491 WARN returning error on flush!
Sep 22 23:24:40.491 DEBG Flush     :1327 extent_limit None deps:[JobId(1326), JobId(1325), JobId(1324)] res:false f:137 g:1
Sep 22 23:24:40.500 DEBG Flush     :1327 extent_limit None deps:[JobId(1326), JobId(1325), JobId(1324)] res:true f:137 g:1
Sep 22 23:24:40.829 DEBG IO Write 1365 has deps [JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1290), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1207), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:40.829 DEBG up_ds_listen was notified
Sep 22 23:24:40.829 DEBG up_ds_listen process 1365
Sep 22 23:24:40.829 DEBG [A] ack job 1365:366, : downstairs
Sep 22 23:24:40.830 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:40.830 DEBG IO Flush 1366 has deps [JobId(1365), JobId(1364), JobId(1363)]
Sep 22 23:24:41.159 DEBG IO Write 1367 has deps [JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1292), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1208), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:41.160 DEBG up_ds_listen was notified
Sep 22 23:24:41.160 DEBG up_ds_listen process 1367
Sep 22 23:24:41.160 DEBG [A] ack job 1367:368, : downstairs
Sep 22 23:24:41.160 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:41.490 DEBG IO Write 1368 has deps [JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1293), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1210), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:41.490 DEBG up_ds_listen was notified
Sep 22 23:24:41.490 DEBG up_ds_listen process 1368
Sep 22 23:24:41.490 DEBG [A] ack job 1368:369, : downstairs
Sep 22 23:24:41.490 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:41.491 DEBG IO Flush 1369 has deps [JobId(1368), JobId(1367), JobId(1366)]
Sep 22 23:24:41.820 DEBG IO Write 1370 has deps [JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1295), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1211), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:41.820 DEBG up_ds_listen was notified
Sep 22 23:24:41.820 DEBG up_ds_listen process 1370
Sep 22 23:24:41.820 DEBG [A] ack job 1370:371, : downstairs
Sep 22 23:24:41.820 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:42.150 DEBG IO Write 1371 has deps [JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1297), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1213), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:42.150 DEBG up_ds_listen was notified
Sep 22 23:24:42.150 DEBG up_ds_listen process 1371
Sep 22 23:24:42.151 DEBG [A] ack job 1371:372, : downstairs
Sep 22 23:24:42.151 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:42.151 DEBG IO Flush 1372 has deps [JobId(1371), JobId(1370), JobId(1369)]
Sep 22 23:24:42.481 DEBG IO Write 1373 has deps [JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1299), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1214), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:42.481 DEBG up_ds_listen was notified
Sep 22 23:24:42.481 DEBG up_ds_listen process 1373
Sep 22 23:24:42.481 DEBG [A] ack job 1373:374, : downstairs
Sep 22 23:24:42.481 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:42.811 DEBG IO Write 1374 has deps [JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1301), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1216), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:42.811 DEBG up_ds_listen was notified
Sep 22 23:24:42.811 DEBG up_ds_listen process 1374
Sep 22 23:24:42.811 DEBG [A] ack job 1374:375, : downstairs
Sep 22 23:24:42.812 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:42.812 DEBG IO Flush 1375 has deps [JobId(1374), JobId(1373), JobId(1372)]
Sep 22 23:24:43.141 DEBG IO Write 1376 has deps [JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1302), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1217), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:43.142 DEBG up_ds_listen was notified
Sep 22 23:24:43.142 DEBG up_ds_listen process 1376
Sep 22 23:24:43.142 DEBG [A] ack job 1376:377, : downstairs
Sep 22 23:24:43.142 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:43.472 DEBG IO Write 1377 has deps [JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1304), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1219), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:43.472 DEBG up_ds_listen was notified
Sep 22 23:24:43.472 DEBG up_ds_listen process 1377
Sep 22 23:24:43.472 DEBG [A] ack job 1377:378, : downstairs
Sep 22 23:24:43.472 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:43.473 DEBG IO Flush 1378 has deps [JobId(1377), JobId(1376), JobId(1375)]
Sep 22 23:24:43.824 DEBG IO Write 1379 has deps [JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1305), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1220), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:43.824 DEBG up_ds_listen was notified
Sep 22 23:24:43.824 DEBG up_ds_listen process 1379
Sep 22 23:24:43.824 DEBG [A] ack job 1379:380, : downstairs
Sep 22 23:24:43.824 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:43.901 DEBG Write     :1328 deps:[JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1241), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1168), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:44.231 DEBG IO Write 1380 has deps [JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1307), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1222), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:44.232 DEBG up_ds_listen was notified
Sep 22 23:24:44.232 DEBG up_ds_listen process 1380
Sep 22 23:24:44.232 DEBG [A] ack job 1380:381, : downstairs
Sep 22 23:24:44.232 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:44.232 DEBG IO Flush 1381 has deps [JobId(1380), JobId(1379), JobId(1378)]
Sep 22 23:24:44.561 DEBG IO Write 1382 has deps [JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1309), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1223), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:44.562 DEBG up_ds_listen was notified
Sep 22 23:24:44.562 DEBG up_ds_listen process 1382
Sep 22 23:24:44.562 DEBG [A] ack job 1382:383, : downstairs
Sep 22 23:24:44.562 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:44.892 DEBG IO Write 1383 has deps [JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1311), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1225), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:44.892 DEBG up_ds_listen was notified
Sep 22 23:24:44.892 DEBG up_ds_listen process 1383
Sep 22 23:24:44.892 DEBG [A] ack job 1383:384, : downstairs
Sep 22 23:24:44.892 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:44.893 DEBG IO Flush 1384 has deps [JobId(1383), JobId(1382), JobId(1381)]
Sep 22 23:24:45.223 DEBG IO Write 1385 has deps [JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1313), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1226), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:45.223 DEBG up_ds_listen was notified
Sep 22 23:24:45.223 DEBG up_ds_listen process 1385
Sep 22 23:24:45.223 DEBG [A] ack job 1385:386, : downstairs
Sep 22 23:24:45.223 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:45.553 DEBG IO Write 1386 has deps [JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1315), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1228), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:45.553 DEBG up_ds_listen was notified
Sep 22 23:24:45.553 DEBG up_ds_listen process 1386
Sep 22 23:24:45.553 DEBG [A] ack job 1386:387, : downstairs
Sep 22 23:24:45.553 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:45.554 DEBG IO Flush 1387 has deps [JobId(1386), JobId(1385), JobId(1384)]
Sep 22 23:24:45.883 DEBG IO Write 1388 has deps [JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1317), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1229), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:45.883 DEBG up_ds_listen was notified
Sep 22 23:24:45.883 DEBG up_ds_listen process 1388
Sep 22 23:24:45.883 DEBG [A] ack job 1388:389, : downstairs
Sep 22 23:24:45.884 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:46.214 DEBG IO Write 1389 has deps [JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1319), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1231), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:46.214 DEBG up_ds_listen was notified
Sep 22 23:24:46.214 DEBG up_ds_listen process 1389
Sep 22 23:24:46.214 DEBG [A] ack job 1389:390, : downstairs
Sep 22 23:24:46.214 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:46.215 DEBG IO Flush 1390 has deps [JobId(1389), JobId(1388), JobId(1387)]
Sep 22 23:24:46.544 DEBG IO Write 1391 has deps [JobId(1390), JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1320), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1232), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)]
Sep 22 23:24:46.545 DEBG up_ds_listen was notified
Sep 22 23:24:46.545 DEBG up_ds_listen process 1391
Sep 22 23:24:46.545 DEBG [A] ack job 1391:392, : downstairs
Sep 22 23:24:46.545 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:46.895 DEBG IO Write 1392 has deps [JobId(1390), JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1322), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1234), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162), JobId(1161)]
Sep 22 23:24:46.896 DEBG up_ds_listen was notified
Sep 22 23:24:46.896 DEBG up_ds_listen process 1392
Sep 22 23:24:46.896 DEBG [A] ack job 1392:393, : downstairs
Sep 22 23:24:46.896 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:46.943 DEBG IO Flush 1393 has deps [JobId(1392), JobId(1391), JobId(1390)]
Sep 22 23:24:46.943 INFO [lossy] sleeping 1 second
Sep 22 23:24:46.944 DEBG IO Read  1394 has deps [JobId(1393)]
Sep 22 23:24:47.172 ERRO [2] job id 1323 saw error GenericError("test error")
Sep 22 23:24:47.172 ERRO [2] job id 1323 saw error GenericError("test error")
Sep 22 23:24:47.172 ERRO [2] job id 1323 saw error GenericError("test error")
Sep 22 23:24:47.172 ERRO [2] job id 1327 saw error GenericError("test error")
Sep 22 23:24:47.494 DEBG IO Flush 1395 has deps [JobId(1394), JobId(1393)]
Sep 22 23:24:48.005 DEBG Write     :1329 deps:[JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1243), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1169), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.015 DEBG Flush     :1330 extent_limit None deps:[JobId(1329), JobId(1328), JobId(1327)] res:true f:138 g:1
Sep 22 23:24:48.015 INFO [lossy] skipping 1331
Sep 22 23:24:48.044 DEBG Write     :1332 deps:[JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1247), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1172), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.045 INFO [lossy] skipping 1335
Sep 22 23:24:48.045 INFO [lossy] skipping 1346
Sep 22 23:24:48.045 INFO [lossy] skipping 1331
Sep 22 23:24:48.074 DEBG Write     :1331 deps:[JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1245), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1171), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.083 DEBG Flush     :1333 extent_limit None deps:[JobId(1332), JobId(1331), JobId(1330)] res:true f:139 g:1
Sep 22 23:24:48.083 INFO [lossy] skipping 1334
Sep 22 23:24:48.084 WARN returning error on write!
Sep 22 23:24:48.084 DEBG Write     :1335 deps:[JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1251), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1175), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.084 INFO [lossy] skipping 1336
Sep 22 23:24:48.084 INFO [lossy] skipping 1337
Sep 22 23:24:48.084 WARN 1338 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.084 INFO [lossy] skipping 1340
Sep 22 23:24:48.084 WARN 1341 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:48.084 INFO [lossy] skipping 1343
Sep 22 23:24:48.084 WARN 1344 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:48.084 WARN 1346 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:48.084 INFO [lossy] skipping 1347
Sep 22 23:24:48.084 WARN 1349 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:48.084 WARN 1350 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:48.084 WARN 1352 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:24:48.086 WARN returning error on write!
Sep 22 23:24:48.086 DEBG Write     :1334 deps:[JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1249), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1174), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.087 WARN returning error on write!
Sep 22 23:24:48.087 DEBG Write     :1335 deps:[JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1251), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1175), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.087 WARN 1336 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:48.087 WARN 1337 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.087 INFO [lossy] skipping 1340
Sep 22 23:24:48.087 WARN 1343 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:48.087 WARN 1347 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:48.088 WARN returning error on write!
Sep 22 23:24:48.088 DEBG Write     :1334 deps:[JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1249), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1174), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.117 DEBG Write     :1335 deps:[JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1251), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1175), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.119 WARN 1340 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:48.148 DEBG Write     :1334 deps:[JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1249), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1174), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.157 DEBG Flush     :1336 extent_limit None deps:[JobId(1335), JobId(1334), JobId(1333)] res:true f:140 g:1
Sep 22 23:24:48.186 DEBG Write     :1337 deps:[JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1253), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1177), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.217 DEBG Write     :1338 deps:[JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1255), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1179), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.226 DEBG Flush     :1339 extent_limit None deps:[JobId(1338), JobId(1337), JobId(1336)] res:true f:141 g:1
Sep 22 23:24:48.255 DEBG Write     :1340 deps:[JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1257), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1180), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.257 WARN returning error on write!
Sep 22 23:24:48.258 DEBG Write     :1341 deps:[JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1259), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1182), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.258 WARN 1342 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.258 WARN 1343 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.258 WARN 1344 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.258 WARN 1346 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:48.258 WARN 1347 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:48.258 INFO [lossy] skipping 1348
Sep 22 23:24:48.258 WARN 1349 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:48.258 WARN 1350 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:48.258 INFO [lossy] skipping 1352
Sep 22 23:24:48.258 INFO [lossy] skipping 1341
Sep 22 23:24:48.258 WARN 1352 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:48.287 DEBG Write     :1341 deps:[JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1259), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1182), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.296 DEBG Flush     :1342 extent_limit None deps:[JobId(1341), JobId(1340), JobId(1339)] res:true f:142 g:1
Sep 22 23:24:48.326 DEBG Write     :1343 deps:[JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1261), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1183), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.327 INFO [lossy] skipping 1344
Sep 22 23:24:48.327 WARN 1345 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.327 WARN 1346 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.327 INFO [lossy] skipping 1347
Sep 22 23:24:48.327 INFO [lossy] skipping 1349
Sep 22 23:24:48.327 WARN 1350 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:48.327 INFO [lossy] skipping 1351
Sep 22 23:24:48.327 WARN 1352 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:48.328 WARN returning error on write!
Sep 22 23:24:48.328 DEBG Write     :1344 deps:[JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1263), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1185), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.328 INFO [lossy] skipping 1347
Sep 22 23:24:48.328 WARN 1349 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:48.329 WARN returning error on write!
Sep 22 23:24:48.329 DEBG Write     :1344 deps:[JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1263), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1185), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.329 INFO [lossy] skipping 1347
Sep 22 23:24:48.330 WARN returning error on write!
Sep 22 23:24:48.330 DEBG Write     :1344 deps:[JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1263), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1185), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.330 WARN 1347 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.330 INFO [lossy] skipping 1344
Sep 22 23:24:48.331 WARN returning error on write!
Sep 22 23:24:48.331 DEBG Write     :1344 deps:[JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1263), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1185), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.331 INFO [lossy] skipping 1344
Sep 22 23:24:48.331 INFO [lossy] skipping 1344
Sep 22 23:24:48.332 WARN returning error on write!
Sep 22 23:24:48.332 DEBG Write     :1344 deps:[JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1263), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1185), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.361 DEBG Write     :1344 deps:[JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1263), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1185), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.362 INFO [lossy] skipping 1345
Sep 22 23:24:48.362 INFO [lossy] skipping 1348
Sep 22 23:24:48.362 INFO [lossy] skipping 1352
Sep 22 23:24:48.362 WARN returning error on flush!
Sep 22 23:24:48.362 DEBG Flush     :1345 extent_limit None deps:[JobId(1344), JobId(1343), JobId(1342)] res:false f:143 g:1
Sep 22 23:24:48.362 INFO [lossy] skipping 1352
Sep 22 23:24:48.370 DEBG Flush     :1345 extent_limit None deps:[JobId(1344), JobId(1343), JobId(1342)] res:true f:143 g:1
Sep 22 23:24:48.370 INFO [lossy] skipping 1352
Sep 22 23:24:48.370 WARN 1352 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:48.370 INFO [lossy] skipping 1346
Sep 22 23:24:48.400 DEBG Write     :1347 deps:[JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1267), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1188), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.401 WARN 1348 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.401 WARN 1349 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.401 WARN 1350 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.401 INFO [lossy] skipping 1352
Sep 22 23:24:48.402 WARN returning error on write!
Sep 22 23:24:48.402 DEBG Write     :1346 deps:[JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1265), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1186), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:48.431 DEBG Write     :1346 deps:[JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1265), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1186), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.441 DEBG Flush     :1348 extent_limit None deps:[JobId(1347), JobId(1346), JobId(1345)] res:true f:144 g:1
Sep 22 23:24:48.441 INFO [lossy] skipping 1349
Sep 22 23:24:48.471 DEBG Write     :1350 deps:[JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1271), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1191), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.472 INFO [lossy] skipping 1351
Sep 22 23:24:48.472 WARN 1352 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:48.502 DEBG Write     :1349 deps:[JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1269), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1189), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:48.512 DEBG Flush     :1351 extent_limit None deps:[JobId(1350), JobId(1349), JobId(1348)] res:true f:145 g:1
Sep 22 23:24:48.512 INFO [lossy] sleeping 1 second
Sep 22 23:24:49.546 WARN returning error on write!
Sep 22 23:24:49.546 DEBG Write     :1352 deps:[JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1273), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1193), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:49.546 INFO [lossy] skipping 1353
Sep 22 23:24:49.546 INFO [lossy] skipping 1354
Sep 22 23:24:49.546 INFO [lossy] skipping 1356
Sep 22 23:24:49.546 INFO [lossy] skipping 1360
Sep 22 23:24:49.546 INFO [lossy] skipping 1364
Sep 22 23:24:49.546 INFO [lossy] skipping 1366
Sep 22 23:24:49.546 INFO [lossy] skipping 1368
Sep 22 23:24:49.546 INFO [lossy] skipping 1374
Sep 22 23:24:49.546 INFO [lossy] skipping 1375
Sep 22 23:24:49.547 WARN returning error on write!
Sep 22 23:24:49.547 DEBG Write     :1352 deps:[JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1273), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1193), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:49.578 DEBG Write     :1353 deps:[JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1275), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1194), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:49.579 INFO [lossy] skipping 1366
Sep 22 23:24:49.579 INFO [lossy] skipping 1374
Sep 22 23:24:49.579 WARN returning error on write!
Sep 22 23:24:49.579 DEBG Write     :1352 deps:[JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1273), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1193), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:49.609 DEBG Write     :1352 deps:[JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1273), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1193), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:49.619 DEBG Flush     :1354 extent_limit None deps:[JobId(1353), JobId(1352), JobId(1351)] res:true f:146 g:1
Sep 22 23:24:49.619 INFO [lossy] skipping 1355
Sep 22 23:24:49.619 INFO [lossy] skipping 1356
Sep 22 23:24:49.619 INFO [lossy] skipping 1357
Sep 22 23:24:49.619 WARN 1358 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:49.619 INFO [lossy] skipping 1359
Sep 22 23:24:49.619 WARN 1361 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:49.619 WARN 1362 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:49.619 INFO [lossy] skipping 1364
Sep 22 23:24:49.619 WARN 1365 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:49.619 WARN 1367 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:49.619 WARN 1368 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:49.619 WARN 1370 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:49.619 WARN 1371 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:49.619 INFO [lossy] skipping 1372
Sep 22 23:24:49.619 WARN 1373 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:24:49.619 INFO [lossy] skipping 1374
Sep 22 23:24:49.619 INFO [lossy] skipping 1355
Sep 22 23:24:49.619 INFO [lossy] skipping 1356
Sep 22 23:24:49.619 WARN 1357 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:49.619 WARN 1359 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:49.619 WARN 1364 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:49.619 INFO [lossy] skipping 1372
Sep 22 23:24:49.619 WARN 1374 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:24:49.649 DEBG Write     :1355 deps:[JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1277), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1196), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:49.680 DEBG Write     :1356 deps:[JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1279), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1197), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:49.681 INFO [lossy] sleeping 1 second
Sep 22 23:24:50.484 ERRO [2] job id 1335 saw error GenericError("test error")
Sep 22 23:24:50.484 ERRO [2] job id 1334 saw error GenericError("test error")
Sep 22 23:24:50.484 ERRO [2] job id 1335 saw error GenericError("test error")
Sep 22 23:24:50.484 ERRO [2] job id 1334 saw error GenericError("test error")
Sep 22 23:24:50.484 ERRO [2] job id 1341 saw error GenericError("test error")
Sep 22 23:24:50.484 ERRO [2] job id 1344 saw error GenericError("test error")
Sep 22 23:24:50.484 ERRO [2] job id 1344 saw error GenericError("test error")
Sep 22 23:24:50.484 ERRO [2] job id 1344 saw error GenericError("test error")
Sep 22 23:24:50.484 ERRO [2] job id 1344 saw error GenericError("test error")
Sep 22 23:24:50.484 ERRO [2] job id 1344 saw error GenericError("test error")
Sep 22 23:24:50.485 ERRO [2] job id 1345 saw error GenericError("test error")
Sep 22 23:24:50.485 ERRO [2] job id 1346 saw error GenericError("test error")
Sep 22 23:24:50.485 ERRO [2] job id 1352 saw error GenericError("test error")
Sep 22 23:24:50.485 ERRO [2] job id 1352 saw error GenericError("test error")
Sep 22 23:24:50.485 ERRO [2] job id 1352 saw error GenericError("test error")
Sep 22 23:24:50.691 DEBG Flush     :1357 extent_limit None deps:[JobId(1356), JobId(1355), JobId(1354)] res:true f:147 g:1
Sep 22 23:24:50.721 DEBG Write     :1358 deps:[JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1281), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1199), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:50.752 DEBG Write     :1359 deps:[JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1283), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1200), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:50.761 DEBG Flush     :1360 extent_limit None deps:[JobId(1359), JobId(1358), JobId(1357)] res:true f:148 g:1
Sep 22 23:24:50.791 DEBG Write     :1361 deps:[JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1285), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1202), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:50.793 WARN returning error on write!
Sep 22 23:24:50.793 DEBG Write     :1362 deps:[JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1287), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1203), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:50.793 INFO [lossy] skipping 1363
Sep 22 23:24:50.793 WARN 1364 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:50.793 INFO [lossy] skipping 1365
Sep 22 23:24:50.793 INFO [lossy] skipping 1366
Sep 22 23:24:50.793 INFO [lossy] skipping 1367
Sep 22 23:24:50.793 WARN 1368 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:50.793 WARN 1370 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:50.793 INFO [lossy] skipping 1371
Sep 22 23:24:50.793 WARN 1373 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:50.794 WARN 1374 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:50.794 INFO [lossy] skipping 1375
Sep 22 23:24:50.794 INFO [lossy] skipping 1376
Sep 22 23:24:50.794 INFO [lossy] skipping 1377
Sep 22 23:24:50.794 INFO [lossy] skipping 1380
Sep 22 23:24:50.794 INFO [lossy] skipping 1383
Sep 22 23:24:50.794 INFO [lossy] skipping 1392
Sep 22 23:24:50.823 DEBG Write     :1362 deps:[JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1287), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1203), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:50.832 DEBG Flush     :1363 extent_limit None deps:[JobId(1362), JobId(1361), JobId(1360)] res:true f:149 g:1
Sep 22 23:24:50.862 DEBG Write     :1365 deps:[JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1290), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1207), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:50.863 WARN 1366 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:50.863 WARN 1367 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:50.863 WARN 1371 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:50.863 INFO [lossy] skipping 1375
Sep 22 23:24:50.863 INFO [lossy] skipping 1376
Sep 22 23:24:50.863 INFO [lossy] skipping 1377
Sep 22 23:24:50.863 INFO [lossy] skipping 1380
Sep 22 23:24:50.863 INFO [lossy] skipping 1375
Sep 22 23:24:50.863 INFO [lossy] skipping 1376
Sep 22 23:24:50.863 INFO [lossy] sleeping 1 second
Sep 22 23:24:50.864 ERRO [2] job id 1362 saw error GenericError("test error")
Sep 22 23:24:51.866 WARN returning error on write!
Sep 22 23:24:51.866 DEBG Write     :1364 deps:[JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1288), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1205), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:51.866 INFO [lossy] skipping 1366
Sep 22 23:24:51.866 WARN 1368 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:51.866 INFO [lossy] skipping 1369
Sep 22 23:24:51.866 WARN 1370 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:51.866 WARN 1373 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:51.866 WARN 1374 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:51.866 INFO [lossy] skipping 1376
Sep 22 23:24:51.866 INFO [lossy] skipping 1377
Sep 22 23:24:51.866 INFO [lossy] skipping 1378
Sep 22 23:24:51.866 WARN 1379 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:51.866 WARN 1382 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:24:51.866 INFO [lossy] skipping 1385
Sep 22 23:24:51.866 INFO [lossy] skipping 1386
Sep 22 23:24:51.866 INFO [lossy] skipping 1388
Sep 22 23:24:51.866 WARN 1389 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:24:51.866 WARN 1391 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 9 deps, role: work
Sep 22 23:24:51.866 INFO [lossy] skipping 1394
Sep 22 23:24:51.866 INFO [lossy] skipping 1395
Sep 22 23:24:51.896 DEBG Write     :1364 deps:[JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1288), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1205), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:51.905 DEBG Flush     :1366 extent_limit None deps:[JobId(1365), JobId(1364), JobId(1363)] res:true f:150 g:1
Sep 22 23:24:51.905 WARN 1369 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:51.905 WARN 1376 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:51.905 WARN 1377 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:51.905 INFO [lossy] skipping 1378
Sep 22 23:24:51.905 WARN 1385 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:24:51.905 WARN 1386 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:24:51.906 WARN 1388 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:24:51.935 DEBG Write     :1367 deps:[JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1292), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1208), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:51.938 WARN returning error on write!
Sep 22 23:24:51.938 DEBG Write     :1368 deps:[JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1293), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1210), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:51.938 INFO [lossy] skipping 1369
Sep 22 23:24:51.938 WARN 1370 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:51.938 WARN 1371 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:51.938 WARN 1373 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:51.938 WARN 1374 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:51.938 INFO [lossy] skipping 1376
Sep 22 23:24:51.938 INFO [lossy] skipping 1379
Sep 22 23:24:51.938 WARN 1380 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:51.938 WARN 1382 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:51.938 WARN 1383 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:51.938 INFO [lossy] skipping 1384
Sep 22 23:24:51.938 INFO [lossy] skipping 1386
Sep 22 23:24:51.938 INFO [lossy] skipping 1387
Sep 22 23:24:51.938 WARN 1389 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 7 deps, role: work
Sep 22 23:24:51.938 WARN 1391 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:24:51.938 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 8 deps, role: work
Sep 22 23:24:51.938 INFO [lossy] skipping 1368
Sep 22 23:24:51.938 WARN 1369 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:51.938 WARN 1379 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:51.938 INFO [lossy] skipping 1387
Sep 22 23:24:51.938 INFO [lossy] skipping 1368
Sep 22 23:24:51.968 DEBG Write     :1368 deps:[JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1293), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1210), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:51.969 INFO [lossy] sleeping 1 second
Sep 22 23:24:51.969 ERRO [2] job id 1364 saw error GenericError("test error")
Sep 22 23:24:51.969 ERRO [2] job id 1368 saw error GenericError("test error")
Sep 22 23:24:52.979 DEBG Flush     :1369 extent_limit None deps:[JobId(1368), JobId(1367), JobId(1366)] res:true f:151 g:1
Sep 22 23:24:53.009 DEBG Write     :1370 deps:[JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1295), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1211), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:53.041 DEBG Write     :1371 deps:[JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1297), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1213), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:53.050 DEBG Flush     :1372 extent_limit None deps:[JobId(1371), JobId(1370), JobId(1369)] res:true f:152 g:1
Sep 22 23:24:53.050 INFO [lossy] skipping 1373
Sep 22 23:24:53.051 WARN returning error on write!
Sep 22 23:24:53.051 DEBG Write     :1374 deps:[JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1301), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1216), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:53.052 INFO [lossy] skipping 1375
Sep 22 23:24:53.052 WARN 1376 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:53.052 WARN 1377 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:53.052 WARN 1379 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:53.052 INFO [lossy] skipping 1380
Sep 22 23:24:53.052 WARN 1382 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:53.052 WARN 1383 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:53.052 WARN 1385 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:53.052 INFO [lossy] skipping 1386
Sep 22 23:24:53.052 INFO [lossy] skipping 1388
Sep 22 23:24:53.052 WARN 1389 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:53.052 INFO [lossy] skipping 1390
Sep 22 23:24:53.052 WARN 1391 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:24:53.052 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 6 deps, role: work
Sep 22 23:24:53.052 INFO [lossy] skipping 1393
Sep 22 23:24:53.052 INFO [lossy] skipping 1373
Sep 22 23:24:53.082 DEBG Write     :1374 deps:[JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1301), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1216), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:53.083 WARN 1375 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:53.083 WARN 1380 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:53.083 INFO [lossy] skipping 1386
Sep 22 23:24:53.083 WARN 1388 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:53.113 DEBG Write     :1373 deps:[JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1299), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1214), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:53.114 WARN 1386 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:53.114 INFO [lossy] sleeping 1 second
Sep 22 23:24:53.115 ERRO [2] job id 1374 saw error GenericError("test error")
Sep 22 23:24:54.115 WARN returning error on flush!
Sep 22 23:24:54.115 DEBG Flush     :1375 extent_limit None deps:[JobId(1374), JobId(1373), JobId(1372)] res:false f:153 g:1
Sep 22 23:24:54.115 INFO [lossy] skipping 1381
Sep 22 23:24:54.115 INFO [lossy] skipping 1384
Sep 22 23:24:54.115 INFO [lossy] skipping 1385
Sep 22 23:24:54.115 INFO [lossy] skipping 1386
Sep 22 23:24:54.115 INFO [lossy] skipping 1388
Sep 22 23:24:54.124 DEBG Flush     :1375 extent_limit None deps:[JobId(1374), JobId(1373), JobId(1372)] res:true f:153 g:1
Sep 22 23:24:54.124 WARN 1385 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:54.124 WARN 1386 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:54.124 WARN 1388 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:54.125 WARN returning error on write!
Sep 22 23:24:54.125 DEBG Write     :1376 deps:[JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1302), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1217), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:54.156 DEBG Write     :1377 deps:[JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1304), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1219), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:54.157 INFO [lossy] skipping 1378
Sep 22 23:24:54.157 WARN 1379 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:54.158 INFO [lossy] skipping 1380
Sep 22 23:24:54.158 WARN 1382 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:54.158 WARN 1383 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:54.158 INFO [lossy] skipping 1386
Sep 22 23:24:54.158 INFO [lossy] skipping 1387
Sep 22 23:24:54.158 INFO [lossy] skipping 1389
Sep 22 23:24:54.158 WARN 1391 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:54.158 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 5 deps, role: work
Sep 22 23:24:54.158 WARN returning error on write!
Sep 22 23:24:54.158 DEBG Write     :1376 deps:[JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1302), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1217), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:54.159 WARN 1378 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:54.159 INFO [lossy] skipping 1380
Sep 22 23:24:54.159 WARN 1389 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:54.188 DEBG Write     :1376 deps:[JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1302), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1217), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:54.189 WARN 1380 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:54.197 DEBG Flush     :1378 extent_limit None deps:[JobId(1377), JobId(1376), JobId(1375)] res:true f:154 g:1
Sep 22 23:24:54.198 WARN returning error on write!
Sep 22 23:24:54.198 DEBG Write     :1379 deps:[JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1305), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1220), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:54.200 WARN returning error on write!
Sep 22 23:24:54.200 DEBG Write     :1380 deps:[JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1307), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1222), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:54.200 WARN 1381 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:54.200 INFO [lossy] skipping 1382
Sep 22 23:24:54.200 INFO [lossy] skipping 1383
Sep 22 23:24:54.200 INFO [lossy] skipping 1384
Sep 22 23:24:54.200 INFO [lossy] skipping 1385
Sep 22 23:24:54.200 WARN 1386 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:54.200 WARN 1388 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:54.200 WARN 1389 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:54.200 WARN 1391 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:54.200 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 4 deps, role: work
Sep 22 23:24:54.200 INFO [lossy] skipping 1394
Sep 22 23:24:54.230 DEBG Write     :1379 deps:[JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1305), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1220), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:54.261 DEBG Write     :1380 deps:[JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1307), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1222), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:54.262 WARN 1382 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:54.262 WARN 1383 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:54.262 WARN 1385 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:54.262 INFO [lossy] skipping 1381
Sep 22 23:24:54.262 INFO [lossy] skipping 1383
Sep 22 23:24:54.262 INFO [lossy] skipping 1386
Sep 22 23:24:54.262 INFO [lossy] skipping 1387
Sep 22 23:24:54.262 INFO [lossy] skipping 1394
Sep 22 23:24:54.262 INFO [lossy] skipping 1395
Sep 22 23:24:54.270 DEBG Flush     :1381 extent_limit None deps:[JobId(1380), JobId(1379), JobId(1378)] res:true f:155 g:1
Sep 22 23:24:54.300 DEBG Write     :1383 deps:[JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1311), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1225), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:54.301 WARN 1386 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:54.331 DEBG Write     :1382 deps:[JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1309), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1223), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:54.332 INFO [lossy] skipping 1384
Sep 22 23:24:54.332 INFO [lossy] skipping 1385
Sep 22 23:24:54.332 INFO [lossy] skipping 1387
Sep 22 23:24:54.332 WARN 1388 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:54.332 WARN 1389 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:54.332 INFO [lossy] skipping 1391
Sep 22 23:24:54.332 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:54.332 INFO [lossy] skipping 1384
Sep 22 23:24:54.332 INFO [lossy] skipping 1385
Sep 22 23:24:54.332 WARN 1391 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 3 deps, role: work
Sep 22 23:24:54.332 WARN returning error on flush!
Sep 22 23:24:54.332 DEBG Flush     :1384 extent_limit None deps:[JobId(1383), JobId(1382), JobId(1381)] res:false f:156 g:1
Sep 22 23:24:54.332 WARN 1385 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:54.332 WARN returning error on flush!
Sep 22 23:24:54.332 DEBG Flush     :1384 extent_limit None deps:[JobId(1383), JobId(1382), JobId(1381)] res:false f:156 g:1
Sep 22 23:24:54.340 DEBG Flush     :1384 extent_limit None deps:[JobId(1383), JobId(1382), JobId(1381)] res:true f:156 g:1
Sep 22 23:24:54.340 INFO [lossy] sleeping 1 second
Sep 22 23:24:54.341 ERRO [2] job id 1375 saw error GenericError("test error")
Sep 22 23:24:54.341 ERRO [2] job id 1376 saw error GenericError("test error")
Sep 22 23:24:54.341 ERRO [2] job id 1376 saw error GenericError("test error")
Sep 22 23:24:54.341 ERRO [2] job id 1379 saw error GenericError("test error")
Sep 22 23:24:54.341 ERRO [2] job id 1380 saw error GenericError("test error")
Sep 22 23:24:54.341 ERRO [2] job id 1384 saw error GenericError("test error")
Sep 22 23:24:54.341 ERRO [2] job id 1384 saw error GenericError("test error")
Sep 22 23:24:55.372 DEBG Write     :1385 deps:[JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1313), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1227), JobId(1226), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:55.403 DEBG Write     :1386 deps:[JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1316), JobId(1315), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1228), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:55.404 WARN returning error on flush!
Sep 22 23:24:55.404 DEBG Flush     :1387 extent_limit None deps:[JobId(1386), JobId(1385), JobId(1384)] res:false f:157 g:1
Sep 22 23:24:55.404 WARN 1388 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:55.404 WARN 1389 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:55.404 INFO [lossy] skipping 1390
Sep 22 23:24:55.404 WARN 1391 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:55.404 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:55.404 INFO [lossy] skipping 1393
Sep 22 23:24:55.404 INFO [lossy] skipping 1395
Sep 22 23:24:55.412 DEBG Flush     :1387 extent_limit None deps:[JobId(1386), JobId(1385), JobId(1384)] res:true f:157 g:1
Sep 22 23:24:55.412 INFO [lossy] skipping 1390
Sep 22 23:24:55.412 INFO [lossy] skipping 1395
Sep 22 23:24:55.412 WARN 1390 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 2 deps, role: work
Sep 22 23:24:55.413 WARN returning error on write!
Sep 22 23:24:55.413 DEBG Write     :1388 deps:[JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1317), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1229), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:false
Sep 22 23:24:55.443 DEBG Write     :1389 deps:[JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1319), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1231), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:55.445 WARN 1390 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:55.445 WARN 1391 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:55.445 WARN 1392 job Write for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:24:55.474 DEBG Write     :1388 deps:[JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1318), JobId(1317), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1230), JobId(1229), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:55.475 INFO [lossy] sleeping 1 second
Sep 22 23:24:55.475 ERRO [2] job id 1387 saw error GenericError("test error")
Sep 22 23:24:55.475 ERRO [2] job id 1388 saw error GenericError("test error")
Sep 22 23:24:56.476 WARN returning error on flush!
Sep 22 23:24:56.476 DEBG Flush     :1390 extent_limit None deps:[JobId(1389), JobId(1388), JobId(1387)] res:false f:158 g:1
Sep 22 23:24:56.476 INFO [lossy] skipping 1392
Sep 22 23:24:56.476 INFO [lossy] skipping 1394
Sep 22 23:24:56.476 INFO [lossy] skipping 1390
Sep 22 23:24:56.476 INFO [lossy] skipping 1392
Sep 22 23:24:56.485 DEBG Flush     :1390 extent_limit None deps:[JobId(1389), JobId(1388), JobId(1387)] res:true f:158 g:1
Sep 22 23:24:56.486 WARN returning error on write!
Sep 22 23:24:56.486 DEBG Write     :1392 deps:[JobId(1390), JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1322), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1234), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162), JobId(1161)] res:false
Sep 22 23:24:56.487 INFO [lossy] skipping 1392
Sep 22 23:24:56.487 INFO [lossy] skipping 1392
Sep 22 23:24:56.487 WARN returning error on write!
Sep 22 23:24:56.487 DEBG Write     :1392 deps:[JobId(1390), JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1322), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1234), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162), JobId(1161)] res:false
Sep 22 23:24:56.488 INFO [lossy] skipping 1392
Sep 22 23:24:56.517 DEBG Write     :1392 deps:[JobId(1390), JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1322), JobId(1321), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1234), JobId(1233), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162), JobId(1161)] res:true
Sep 22 23:24:56.548 DEBG Write     :1391 deps:[JobId(1390), JobId(1387), JobId(1384), JobId(1381), JobId(1378), JobId(1375), JobId(1372), JobId(1369), JobId(1366), JobId(1363), JobId(1360), JobId(1357), JobId(1354), JobId(1351), JobId(1348), JobId(1345), JobId(1342), JobId(1339), JobId(1336), JobId(1333), JobId(1330), JobId(1327), JobId(1324), JobId(1321), JobId(1320), JobId(1318), JobId(1316), JobId(1314), JobId(1312), JobId(1310), JobId(1308), JobId(1306), JobId(1303), JobId(1300), JobId(1298), JobId(1296), JobId(1294), JobId(1291), JobId(1289), JobId(1286), JobId(1284), JobId(1282), JobId(1280), JobId(1278), JobId(1276), JobId(1274), JobId(1272), JobId(1270), JobId(1268), JobId(1266), JobId(1264), JobId(1262), JobId(1260), JobId(1258), JobId(1256), JobId(1254), JobId(1252), JobId(1250), JobId(1248), JobId(1246), JobId(1244), JobId(1242), JobId(1240), JobId(1238), JobId(1236), JobId(1233), JobId(1232), JobId(1230), JobId(1227), JobId(1224), JobId(1221), JobId(1218), JobId(1215), JobId(1212), JobId(1209), JobId(1206), JobId(1204), JobId(1201), JobId(1198), JobId(1195), JobId(1192), JobId(1190), JobId(1187), JobId(1184), JobId(1181), JobId(1178), JobId(1176), JobId(1173), JobId(1170), JobId(1167), JobId(1164), JobId(1162)] res:true
Sep 22 23:24:56.549 INFO [lossy] skipping 1393
Sep 22 23:24:56.549 INFO [lossy] skipping 1394
Sep 22 23:24:56.549 INFO [lossy] skipping 1395
Sep 22 23:24:56.558 DEBG Flush     :1393 extent_limit None deps:[JobId(1392), JobId(1391), JobId(1390)] res:true f:159 g:1
Sep 22 23:24:56.564 DEBG Read      :1394 deps:[JobId(1393)] res:true
Sep 22 23:24:56.585 ERRO [2] job id 1390 saw error GenericError("test error")
Sep 22 23:24:56.585 ERRO [2] job id 1392 saw error GenericError("test error")
Sep 22 23:24:56.585 ERRO [2] job id 1392 saw error GenericError("test error")
Sep 22 23:24:56.587 INFO [lossy] skipping 1395
Sep 22 23:24:56.587 DEBG Flush     :1395 extent_limit None deps:[JobId(1394), JobId(1393)] res:true f:160 g:1
Sep 22 23:24:56.587 INFO [lossy] sleeping 1 second
Sep 22 23:24:56.964 DEBG [2] Read AckReady 1394, : downstairs
Sep 22 23:24:56.965 DEBG up_ds_listen was notified
Sep 22 23:24:56.965 DEBG up_ds_listen process 1394
Sep 22 23:24:56.965 DEBG [A] ack job 1394:395, : downstairs
Sep 22 23:24:57.018 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:57.019 DEBG IO Read  1396 has deps [JobId(1395)]
Sep 22 23:24:57.086 DEBG IO Flush 1397 has deps [JobId(1396), JobId(1395)]
Sep 22 23:24:57.588 INFO [lossy] skipping 1396
Sep 22 23:24:57.588 WARN returning error on read!
Sep 22 23:24:57.588 DEBG Read      :1396 deps:[JobId(1395)] res:false
Sep 22 23:24:57.588 INFO [lossy] skipping 1396
Sep 22 23:24:57.588 INFO [lossy] skipping 1396
Sep 22 23:24:57.588 WARN returning error on read!
Sep 22 23:24:57.588 DEBG Read      :1396 deps:[JobId(1395)] res:false
Sep 22 23:24:57.595 DEBG Read      :1396 deps:[JobId(1395)] res:true
Sep 22 23:24:57.616 ERRO [2] job id 1396 saw error GenericError("test error")
Sep 22 23:24:57.616 ERRO [2] job id 1396 saw error GenericError("test error")
Sep 22 23:24:57.618 INFO [lossy] sleeping 1 second
Sep 22 23:24:57.996 DEBG [2] Read AckReady 1396, : downstairs
Sep 22 23:24:57.997 DEBG up_ds_listen was notified
Sep 22 23:24:57.997 DEBG up_ds_listen process 1396
Sep 22 23:24:57.997 DEBG [A] ack job 1396:397, : downstairs
Sep 22 23:24:58.050 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:58.051 DEBG IO Read  1398 has deps [JobId(1397)]
Sep 22 23:24:58.088 DEBG IO Flush 1399 has deps [JobId(1398), JobId(1397)]
Sep 22 23:24:58.620 DEBG Flush     :1397 extent_limit None deps:[JobId(1396), JobId(1395)] res:true f:161 g:1
Sep 22 23:24:58.620 WARN returning error on read!
Sep 22 23:24:58.620 DEBG Read      :1398 deps:[JobId(1397)] res:false
Sep 22 23:24:58.620 INFO [lossy] skipping 1398
Sep 22 23:24:58.626 DEBG Read      :1398 deps:[JobId(1397)] res:true
Sep 22 23:24:58.647 ERRO [2] job id 1398 saw error GenericError("test error")
Sep 22 23:24:58.649 DEBG Flush     :1399 extent_limit None deps:[JobId(1398), JobId(1397)] res:true f:162 g:1
Sep 22 23:24:58.649 INFO [lossy] sleeping 1 second
Sep 22 23:24:59.026 DEBG [2] Read AckReady 1398, : downstairs
Sep 22 23:24:59.027 DEBG up_ds_listen was notified
Sep 22 23:24:59.027 DEBG up_ds_listen process 1398
Sep 22 23:24:59.027 DEBG [A] ack job 1398:399, : downstairs
Sep 22 23:24:59.080 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:24:59.081 DEBG IO Read  1400 has deps [JobId(1399)]
Sep 22 23:24:59.090 DEBG IO Flush 1401 has deps [JobId(1400), JobId(1399)]
Sep 22 23:24:59.657 DEBG Read      :1400 deps:[JobId(1399)] res:true
Sep 22 23:24:59.680 DEBG Flush     :1401 extent_limit None deps:[JobId(1400), JobId(1399)] res:true f:163 g:1
Sep 22 23:24:59.680 INFO [lossy] sleeping 1 second
Sep 22 23:25:00.056 DEBG [2] Read AckReady 1400, : downstairs
Sep 22 23:25:00.057 DEBG up_ds_listen was notified
Sep 22 23:25:00.057 DEBG up_ds_listen process 1400
Sep 22 23:25:00.057 DEBG [A] ack job 1400:401, : downstairs
Sep 22 23:25:00.110 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:00.111 DEBG IO Read  1402 has deps [JobId(1401)]
Sep 22 23:25:00.612 DEBG IO Flush 1403 has deps [JobId(1402), JobId(1401)]
Sep 22 23:25:00.688 DEBG Read      :1402 deps:[JobId(1401)] res:true
Sep 22 23:25:00.711 WARN returning error on flush!
Sep 22 23:25:00.711 DEBG Flush     :1403 extent_limit None deps:[JobId(1402), JobId(1401)] res:false f:164 g:1
Sep 22 23:25:00.711 INFO [lossy] skipping 1403
Sep 22 23:25:00.711 INFO [lossy] skipping 1403
Sep 22 23:25:00.711 DEBG Flush     :1403 extent_limit None deps:[JobId(1402), JobId(1401)] res:true f:164 g:1
Sep 22 23:25:00.711 INFO [lossy] sleeping 1 second
Sep 22 23:25:01.088 DEBG [2] Read AckReady 1402, : downstairs
Sep 22 23:25:01.089 ERRO [2] job id 1403 saw error GenericError("test error")
Sep 22 23:25:01.089 DEBG up_ds_listen was notified
Sep 22 23:25:01.089 DEBG up_ds_listen process 1402
Sep 22 23:25:01.089 DEBG [A] ack job 1402:403, : downstairs
Sep 22 23:25:01.142 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:01.143 DEBG IO Read  1404 has deps [JobId(1403)]
Sep 22 23:25:01.644 DEBG IO Flush 1405 has deps [JobId(1404), JobId(1403)]
Sep 22 23:25:01.713 WARN returning error on read!
Sep 22 23:25:01.713 DEBG Read      :1404 deps:[JobId(1403)] res:false
Sep 22 23:25:01.713 INFO [lossy] skipping 1404
Sep 22 23:25:01.713 INFO [lossy] skipping 1404
Sep 22 23:25:01.719 DEBG Read      :1404 deps:[JobId(1403)] res:true
Sep 22 23:25:01.740 ERRO [2] job id 1404 saw error GenericError("test error")
Sep 22 23:25:01.742 WARN returning error on flush!
Sep 22 23:25:01.742 DEBG Flush     :1405 extent_limit None deps:[JobId(1404), JobId(1403)] res:false f:165 g:1
Sep 22 23:25:01.742 DEBG Flush     :1405 extent_limit None deps:[JobId(1404), JobId(1403)] res:true f:165 g:1
Sep 22 23:25:01.742 INFO [lossy] sleeping 1 second
Sep 22 23:25:02.121 DEBG [2] Read AckReady 1404, : downstairs
Sep 22 23:25:02.122 ERRO [2] job id 1405 saw error GenericError("test error")
Sep 22 23:25:02.122 DEBG up_ds_listen was notified
Sep 22 23:25:02.122 DEBG up_ds_listen process 1404
Sep 22 23:25:02.122 DEBG [A] ack job 1404:405, : downstairs
Sep 22 23:25:02.175 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:02.177 DEBG IO Read  1406 has deps [JobId(1405)]
Sep 22 23:25:02.677 DEBG IO Flush 1407 has deps [JobId(1406), JobId(1405)]
Sep 22 23:25:02.749 DEBG Read      :1406 deps:[JobId(1405)] res:true
Sep 22 23:25:02.772 DEBG Flush     :1407 extent_limit None deps:[JobId(1406), JobId(1405)] res:true f:166 g:1
Sep 22 23:25:02.772 INFO [lossy] sleeping 1 second
Sep 22 23:25:03.150 DEBG [2] Read AckReady 1406, : downstairs
Sep 22 23:25:03.151 DEBG up_ds_listen was notified
Sep 22 23:25:03.151 DEBG up_ds_listen process 1406
Sep 22 23:25:03.151 DEBG [A] ack job 1406:407, : downstairs
Sep 22 23:25:03.203 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:03.205 DEBG IO Read  1408 has deps [JobId(1407)]
Sep 22 23:25:03.706 DEBG IO Flush 1409 has deps [JobId(1408), JobId(1407)]
Sep 22 23:25:03.779 DEBG Read      :1408 deps:[JobId(1407)] res:true
Sep 22 23:25:03.802 DEBG Flush     :1409 extent_limit None deps:[JobId(1408), JobId(1407)] res:true f:167 g:1
Sep 22 23:25:03.802 INFO [lossy] sleeping 1 second
Sep 22 23:25:04.181 DEBG [2] Read AckReady 1408, : downstairs
Sep 22 23:25:04.182 DEBG up_ds_listen was notified
Sep 22 23:25:04.182 DEBG up_ds_listen process 1408
Sep 22 23:25:04.182 DEBG [A] ack job 1408:409, : downstairs
Sep 22 23:25:04.235 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:04.236 DEBG IO Read  1410 has deps [JobId(1409)]
Sep 22 23:25:04.737 DEBG IO Flush 1411 has deps [JobId(1410), JobId(1409)]
Sep 22 23:25:04.811 DEBG Read      :1410 deps:[JobId(1409)] res:true
Sep 22 23:25:04.834 INFO [lossy] skipping 1411
Sep 22 23:25:04.834 DEBG Flush     :1411 extent_limit None deps:[JobId(1410), JobId(1409)] res:true f:168 g:1
Sep 22 23:25:04.834 INFO [lossy] sleeping 1 second
Sep 22 23:25:05.211 DEBG [2] Read AckReady 1410, : downstairs
Sep 22 23:25:05.212 DEBG up_ds_listen was notified
Sep 22 23:25:05.212 DEBG up_ds_listen process 1410
Sep 22 23:25:05.212 DEBG [A] ack job 1410:411, : downstairs
Sep 22 23:25:05.265 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:05.266 DEBG IO Read  1412 has deps [JobId(1411)]
Sep 22 23:25:05.767 DEBG IO Flush 1413 has deps [JobId(1412), JobId(1411)]
Sep 22 23:25:05.835 WARN returning error on read!
Sep 22 23:25:05.835 DEBG Read      :1412 deps:[JobId(1411)] res:false
Sep 22 23:25:05.841 DEBG Read      :1412 deps:[JobId(1411)] res:true
Sep 22 23:25:05.863 ERRO [2] job id 1412 saw error GenericError("test error")
Sep 22 23:25:05.864 DEBG Flush     :1413 extent_limit None deps:[JobId(1412), JobId(1411)] res:true f:169 g:1
Sep 22 23:25:05.864 INFO [lossy] sleeping 1 second
Sep 22 23:25:06.241 DEBG [2] Read AckReady 1412, : downstairs
Sep 22 23:25:06.242 DEBG up_ds_listen was notified
Sep 22 23:25:06.242 DEBG up_ds_listen process 1412
Sep 22 23:25:06.242 DEBG [A] ack job 1412:413, : downstairs
Sep 22 23:25:06.295 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:06.296 DEBG IO Read  1414 has deps [JobId(1413)]
Sep 22 23:25:06.797 DEBG IO Flush 1415 has deps [JobId(1414), JobId(1413)]
Sep 22 23:25:06.872 DEBG Read      :1414 deps:[JobId(1413)] res:true
Sep 22 23:25:06.895 DEBG Flush     :1415 extent_limit None deps:[JobId(1414), JobId(1413)] res:true f:170 g:1
Sep 22 23:25:06.895 INFO [lossy] sleeping 1 second
Sep 22 23:25:07.271 DEBG [2] Read AckReady 1414, : downstairs
Sep 22 23:25:07.273 DEBG up_ds_listen was notified
Sep 22 23:25:07.273 DEBG up_ds_listen process 1414
Sep 22 23:25:07.273 DEBG [A] ack job 1414:415, : downstairs
Sep 22 23:25:07.325 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:07.326 DEBG IO Read  1416 has deps [JobId(1415)]
Sep 22 23:25:07.827 DEBG IO Flush 1417 has deps [JobId(1416), JobId(1415)]
Sep 22 23:25:07.903 DEBG Read      :1416 deps:[JobId(1415)] res:true
Sep 22 23:25:07.926 INFO [lossy] skipping 1417
Sep 22 23:25:07.926 DEBG Flush     :1417 extent_limit None deps:[JobId(1416), JobId(1415)] res:true f:171 g:1
Sep 22 23:25:07.926 INFO [lossy] sleeping 1 second
Sep 22 23:25:08.302 DEBG [2] Read AckReady 1416, : downstairs
Sep 22 23:25:08.303 DEBG up_ds_listen was notified
Sep 22 23:25:08.303 DEBG up_ds_listen process 1416
Sep 22 23:25:08.303 DEBG [A] ack job 1416:417, : downstairs
Sep 22 23:25:08.356 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:08.357 DEBG IO Read  1418 has deps [JobId(1417)]
Sep 22 23:25:08.858 DEBG IO Flush 1419 has deps [JobId(1418), JobId(1417)]
Sep 22 23:25:08.933 DEBG Read      :1418 deps:[JobId(1417)] res:true
Sep 22 23:25:08.956 DEBG Flush     :1419 extent_limit None deps:[JobId(1418), JobId(1417)] res:true f:172 g:1
Sep 22 23:25:08.956 INFO [lossy] sleeping 1 second
Sep 22 23:25:09.334 DEBG [2] Read AckReady 1418, : downstairs
Sep 22 23:25:09.335 DEBG up_ds_listen was notified
Sep 22 23:25:09.335 DEBG up_ds_listen process 1418
Sep 22 23:25:09.335 DEBG [A] ack job 1418:419, : downstairs
Sep 22 23:25:09.388 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:09.389 DEBG IO Read  1420 has deps [JobId(1419)]
Sep 22 23:25:09.890 DEBG IO Flush 1421 has deps [JobId(1420), JobId(1419)]
Sep 22 23:25:09.964 DEBG Read      :1420 deps:[JobId(1419)] res:true
Sep 22 23:25:09.987 DEBG Flush     :1421 extent_limit None deps:[JobId(1420), JobId(1419)] res:true f:173 g:1
Sep 22 23:25:09.987 INFO [lossy] sleeping 1 second
Sep 22 23:25:10.364 DEBG [2] Read AckReady 1420, : downstairs
Sep 22 23:25:10.365 DEBG up_ds_listen was notified
Sep 22 23:25:10.365 DEBG up_ds_listen process 1420
Sep 22 23:25:10.366 DEBG [A] ack job 1420:421, : downstairs
Sep 22 23:25:10.418 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:10.420 DEBG IO Read  1422 has deps [JobId(1421)]
Sep 22 23:25:10.921 DEBG IO Flush 1423 has deps [JobId(1422), JobId(1421)]
Sep 22 23:25:10.994 DEBG Read      :1422 deps:[JobId(1421)] res:true
Sep 22 23:25:11.017 DEBG Flush     :1423 extent_limit None deps:[JobId(1422), JobId(1421)] res:true f:174 g:1
Sep 22 23:25:11.017 INFO [lossy] sleeping 1 second
Sep 22 23:25:11.394 DEBG [2] Read AckReady 1422, : downstairs
Sep 22 23:25:11.395 DEBG up_ds_listen was notified
Sep 22 23:25:11.395 DEBG up_ds_listen process 1422
Sep 22 23:25:11.395 DEBG [A] ack job 1422:423, : downstairs
Sep 22 23:25:11.447 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:11.449 DEBG IO Read  1424 has deps [JobId(1423)]
Sep 22 23:25:11.950 DEBG IO Flush 1425 has deps [JobId(1424), JobId(1423)]
Sep 22 23:25:12.019 INFO [lossy] skipping 1424
Sep 22 23:25:12.026 DEBG Read      :1424 deps:[JobId(1423)] res:true
Sep 22 23:25:12.049 DEBG Flush     :1425 extent_limit None deps:[JobId(1424), JobId(1423)] res:true f:175 g:1
Sep 22 23:25:12.049 INFO [lossy] sleeping 1 second
Sep 22 23:25:12.425 DEBG [2] Read AckReady 1424, : downstairs
Sep 22 23:25:12.426 DEBG up_ds_listen was notified
Sep 22 23:25:12.427 DEBG up_ds_listen process 1424
Sep 22 23:25:12.427 DEBG [A] ack job 1424:425, : downstairs
Sep 22 23:25:12.479 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:12.481 DEBG IO Read  1426 has deps [JobId(1425)]
Sep 22 23:25:12.981 DEBG IO Flush 1427 has deps [JobId(1426), JobId(1425)]
Sep 22 23:25:13.050 WARN returning error on read!
Sep 22 23:25:13.050 DEBG Read      :1426 deps:[JobId(1425)] res:false
Sep 22 23:25:13.056 DEBG Read      :1426 deps:[JobId(1425)] res:true
Sep 22 23:25:13.078 ERRO [2] job id 1426 saw error GenericError("test error")
Sep 22 23:25:13.079 INFO [lossy] skipping 1427
Sep 22 23:25:13.079 DEBG Flush     :1427 extent_limit None deps:[JobId(1426), JobId(1425)] res:true f:176 g:1
Sep 22 23:25:13.079 INFO [lossy] sleeping 1 second
Sep 22 23:25:13.456 DEBG [2] Read AckReady 1426, : downstairs
Sep 22 23:25:13.457 DEBG up_ds_listen was notified
Sep 22 23:25:13.457 DEBG up_ds_listen process 1426
Sep 22 23:25:13.457 DEBG [A] ack job 1426:427, : downstairs
Sep 22 23:25:13.509 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:13.511 DEBG IO Read  1428 has deps [JobId(1427)]
Sep 22 23:25:14.012 DEBG IO Flush 1429 has deps [JobId(1428), JobId(1427)]
Sep 22 23:25:14.086 DEBG Read      :1428 deps:[JobId(1427)] res:true
Sep 22 23:25:14.109 DEBG Flush     :1429 extent_limit None deps:[JobId(1428), JobId(1427)] res:true f:177 g:1
Sep 22 23:25:14.109 INFO [lossy] sleeping 1 second
Sep 22 23:25:14.485 DEBG [2] Read AckReady 1428, : downstairs
Sep 22 23:25:14.486 DEBG up_ds_listen was notified
Sep 22 23:25:14.486 DEBG up_ds_listen process 1428
Sep 22 23:25:14.486 DEBG [A] ack job 1428:429, : downstairs
Sep 22 23:25:14.538 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:14.540 DEBG IO Read  1430 has deps [JobId(1429)]
Sep 22 23:25:15.041 DEBG IO Flush 1431 has deps [JobId(1430), JobId(1429)]
Sep 22 23:25:15.116 DEBG Read      :1430 deps:[JobId(1429)] res:true
Sep 22 23:25:15.139 DEBG Flush     :1431 extent_limit None deps:[JobId(1430), JobId(1429)] res:true f:178 g:1
Sep 22 23:25:15.139 INFO [lossy] sleeping 1 second
Sep 22 23:25:15.515 DEBG [2] Read AckReady 1430, : downstairs
Sep 22 23:25:15.516 DEBG up_ds_listen was notified
Sep 22 23:25:15.516 DEBG up_ds_listen process 1430
Sep 22 23:25:15.516 DEBG [A] ack job 1430:431, : downstairs
Sep 22 23:25:15.569 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:15.570 DEBG IO Read  1432 has deps [JobId(1431)]
Sep 22 23:25:16.070 DEBG IO Flush 1433 has deps [JobId(1432), JobId(1431)]
Sep 22 23:25:16.140 WARN returning error on read!
Sep 22 23:25:16.140 DEBG Read      :1432 deps:[JobId(1431)] res:false
Sep 22 23:25:16.147 DEBG Read      :1432 deps:[JobId(1431)] res:true
Sep 22 23:25:16.168 ERRO [2] job id 1432 saw error GenericError("test error")
Sep 22 23:25:16.170 DEBG Flush     :1433 extent_limit None deps:[JobId(1432), JobId(1431)] res:true f:179 g:1
Sep 22 23:25:16.170 INFO [lossy] sleeping 1 second
Sep 22 23:25:16.547 DEBG [2] Read AckReady 1432, : downstairs
Sep 22 23:25:16.548 DEBG up_ds_listen was notified
Sep 22 23:25:16.548 DEBG up_ds_listen process 1432
Sep 22 23:25:16.548 DEBG [A] ack job 1432:433, : downstairs
Sep 22 23:25:16.600 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:16.601 DEBG IO Read  1434 has deps [JobId(1433)]
Sep 22 23:25:17.102 DEBG IO Flush 1435 has deps [JobId(1434), JobId(1433)]
Sep 22 23:25:17.177 DEBG Read      :1434 deps:[JobId(1433)] res:true
Sep 22 23:25:17.200 INFO [lossy] skipping 1435
Sep 22 23:25:17.200 WARN returning error on flush!
Sep 22 23:25:17.200 DEBG Flush     :1435 extent_limit None deps:[JobId(1434), JobId(1433)] res:false f:180 g:1
Sep 22 23:25:17.200 INFO [lossy] skipping 1435
Sep 22 23:25:17.200 DEBG Flush     :1435 extent_limit None deps:[JobId(1434), JobId(1433)] res:true f:180 g:1
Sep 22 23:25:17.200 INFO [lossy] sleeping 1 second
Sep 22 23:25:17.577 DEBG [2] Read AckReady 1434, : downstairs
Sep 22 23:25:17.578 ERRO [2] job id 1435 saw error GenericError("test error")
Sep 22 23:25:17.578 DEBG up_ds_listen was notified
Sep 22 23:25:17.578 DEBG up_ds_listen process 1434
Sep 22 23:25:17.578 DEBG [A] ack job 1434:435, : downstairs
Sep 22 23:25:17.631 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:17.632 DEBG IO Read  1436 has deps [JobId(1435)]
Sep 22 23:25:18.133 DEBG IO Flush 1437 has deps [JobId(1436), JobId(1435)]
Sep 22 23:25:18.207 DEBG Read      :1436 deps:[JobId(1435)] res:true
Sep 22 23:25:18.230 DEBG Flush     :1437 extent_limit None deps:[JobId(1436), JobId(1435)] res:true f:181 g:1
Sep 22 23:25:18.230 INFO [lossy] sleeping 1 second
Sep 22 23:25:18.606 DEBG [2] Read AckReady 1436, : downstairs
Sep 22 23:25:18.607 DEBG up_ds_listen was notified
Sep 22 23:25:18.607 DEBG up_ds_listen process 1436
Sep 22 23:25:18.608 DEBG [A] ack job 1436:437, : downstairs
Sep 22 23:25:18.660 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:18.661 DEBG IO Read  1438 has deps [JobId(1437)]
Sep 22 23:25:19.163 DEBG IO Flush 1439 has deps [JobId(1438), JobId(1437)]
Sep 22 23:25:19.231 WARN returning error on read!
Sep 22 23:25:19.231 DEBG Read      :1438 deps:[JobId(1437)] res:false
Sep 22 23:25:19.231 INFO [lossy] skipping 1439
Sep 22 23:25:19.237 DEBG Read      :1438 deps:[JobId(1437)] res:true
Sep 22 23:25:19.258 ERRO [2] job id 1438 saw error GenericError("test error")
Sep 22 23:25:19.260 DEBG Flush     :1439 extent_limit None deps:[JobId(1438), JobId(1437)] res:true f:182 g:1
Sep 22 23:25:19.260 INFO [lossy] sleeping 1 second
Sep 22 23:25:19.636 DEBG [2] Read AckReady 1438, : downstairs
Sep 22 23:25:19.637 DEBG up_ds_listen was notified
Sep 22 23:25:19.637 DEBG up_ds_listen process 1438
Sep 22 23:25:19.637 DEBG [A] ack job 1438:439, : downstairs
Sep 22 23:25:19.689 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:19.691 DEBG IO Read  1440 has deps [JobId(1439)]
Sep 22 23:25:20.192 DEBG IO Flush 1441 has deps [JobId(1440), JobId(1439)]
Sep 22 23:25:20.267 DEBG Read      :1440 deps:[JobId(1439)] res:true
Sep 22 23:25:20.290 DEBG Flush     :1441 extent_limit None deps:[JobId(1440), JobId(1439)] res:true f:183 g:1
Sep 22 23:25:20.666 DEBG [2] Read AckReady 1440, : downstairs
Sep 22 23:25:20.667 DEBG up_ds_listen was notified
Sep 22 23:25:20.667 DEBG up_ds_listen process 1440
Sep 22 23:25:20.667 DEBG [A] ack job 1440:441, : downstairs
Sep 22 23:25:20.720 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:20.721 DEBG IO Read  1442 has deps [JobId(1441)]
Sep 22 23:25:20.725 INFO [lossy] skipping 1442
Sep 22 23:25:20.725 INFO [lossy] skipping 1442
Sep 22 23:25:20.732 DEBG Read      :1442 deps:[JobId(1441)] res:true
Sep 22 23:25:21.132 DEBG [2] Read AckReady 1442, : downstairs
Sep 22 23:25:21.133 DEBG up_ds_listen was notified
Sep 22 23:25:21.133 DEBG up_ds_listen process 1442
Sep 22 23:25:21.134 DEBG [A] ack job 1442:443, : downstairs
Sep 22 23:25:21.186 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:21.188 DEBG IO Read  1443 has deps [JobId(1441)]
Sep 22 23:25:21.192 WARN returning error on read!
Sep 22 23:25:21.192 DEBG Read      :1443 deps:[JobId(1441)] res:false
Sep 22 23:25:21.198 DEBG Read      :1443 deps:[JobId(1441)] res:true
Sep 22 23:25:21.219 ERRO [2] job id 1443 saw error GenericError("test error")
Sep 22 23:25:21.598 DEBG [2] Read AckReady 1443, : downstairs
Sep 22 23:25:21.599 DEBG up_ds_listen was notified
Sep 22 23:25:21.599 DEBG up_ds_listen process 1443
Sep 22 23:25:21.599 DEBG [A] ack job 1443:444, : downstairs
Sep 22 23:25:21.652 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:21.653 DEBG IO Flush 1444 has deps [JobId(1443), JobId(1442), JobId(1441)]
Sep 22 23:25:21.654 DEBG IO Read  1445 has deps [JobId(1444)]
Sep 22 23:25:21.656 DEBG Flush     :1444 extent_limit None deps:[JobId(1443), JobId(1442), JobId(1441)] res:true f:184 g:1
Sep 22 23:25:21.658 INFO [lossy] skipping 1445
Sep 22 23:25:21.664 DEBG Read      :1445 deps:[JobId(1444)] res:true
Sep 22 23:25:22.064 DEBG [2] Read AckReady 1445, : downstairs
Sep 22 23:25:22.065 DEBG up_ds_listen was notified
Sep 22 23:25:22.065 DEBG up_ds_listen process 1445
Sep 22 23:25:22.065 DEBG [A] ack job 1445:446, : downstairs
Sep 22 23:25:22.118 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:22.119 DEBG IO Read  1446 has deps [JobId(1444)]
Sep 22 23:25:22.130 DEBG Read      :1446 deps:[JobId(1444)] res:true
Sep 22 23:25:22.530 DEBG [2] Read AckReady 1446, : downstairs
Sep 22 23:25:22.531 DEBG up_ds_listen was notified
Sep 22 23:25:22.531 DEBG up_ds_listen process 1446
Sep 22 23:25:22.531 DEBG [A] ack job 1446:447, : downstairs
Sep 22 23:25:22.583 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:22.584 DEBG IO Flush 1447 has deps [JobId(1446), JobId(1445), JobId(1444)]
Sep 22 23:25:22.585 DEBG IO Read  1448 has deps [JobId(1447)]
Sep 22 23:25:22.587 DEBG Flush     :1447 extent_limit None deps:[JobId(1446), JobId(1445), JobId(1444)] res:true f:185 g:1
Sep 22 23:25:22.596 DEBG Read      :1448 deps:[JobId(1447)] res:true
Sep 22 23:25:22.996 DEBG [2] Read AckReady 1448, : downstairs
Sep 22 23:25:22.997 DEBG up_ds_listen was notified
Sep 22 23:25:22.997 DEBG up_ds_listen process 1448
Sep 22 23:25:22.997 DEBG [A] ack job 1448:449, : downstairs
Sep 22 23:25:23.050 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:23.051 INFO [lossy] sleeping 1 second
Sep 22 23:25:23.051 DEBG IO Read  1449 has deps [JobId(1447)]
Sep 22 23:25:23.085 DEBG IO Flush 1450 has deps [JobId(1449), JobId(1448), JobId(1447)]
Sep 22 23:25:24.058 DEBG Read      :1449 deps:[JobId(1447)] res:true
Sep 22 23:25:24.080 DEBG Flush     :1450 extent_limit None deps:[JobId(1449), JobId(1448), JobId(1447)] res:true f:186 g:1
Sep 22 23:25:24.458 DEBG [2] Read AckReady 1449, : downstairs
Sep 22 23:25:24.460 DEBG up_ds_listen was notified
Sep 22 23:25:24.460 DEBG up_ds_listen process 1449
Sep 22 23:25:24.460 DEBG [A] ack job 1449:450, : downstairs
Sep 22 23:25:24.512 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:24.514 DEBG IO Read  1451 has deps [JobId(1450)]
Sep 22 23:25:24.518 INFO [lossy] skipping 1451
Sep 22 23:25:24.524 DEBG Read      :1451 deps:[JobId(1450)] res:true
Sep 22 23:25:24.923 DEBG [2] Read AckReady 1451, : downstairs
Sep 22 23:25:24.924 DEBG up_ds_listen was notified
Sep 22 23:25:24.924 DEBG up_ds_listen process 1451
Sep 22 23:25:24.924 DEBG [A] ack job 1451:452, : downstairs
Sep 22 23:25:24.977 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:24.978 DEBG IO Read  1452 has deps [JobId(1450)]
Sep 22 23:25:24.988 DEBG Read      :1452 deps:[JobId(1450)] res:true
Sep 22 23:25:25.387 DEBG [2] Read AckReady 1452, : downstairs
Sep 22 23:25:25.388 DEBG up_ds_listen was notified
Sep 22 23:25:25.388 DEBG up_ds_listen process 1452
Sep 22 23:25:25.388 DEBG [A] ack job 1452:453, : downstairs
Sep 22 23:25:25.440 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:25.441 DEBG IO Flush 1453 has deps [JobId(1452), JobId(1451), JobId(1450)]
Sep 22 23:25:25.442 DEBG IO Read  1454 has deps [JobId(1453)]
Sep 22 23:25:25.444 INFO [lossy] sleeping 1 second
Sep 22 23:25:25.943 DEBG IO Flush 1455 has deps [JobId(1454), JobId(1453)]
Sep 22 23:25:26.444 INFO [lossy] skipping 1453
Sep 22 23:25:26.444 INFO [lossy] skipping 1455
Sep 22 23:25:26.444 INFO [lossy] skipping 1453
Sep 22 23:25:26.444 DEBG Flush     :1453 extent_limit None deps:[JobId(1452), JobId(1451), JobId(1450)] res:true f:187 g:1
Sep 22 23:25:26.445 INFO [lossy] sleeping 1 second
Sep 22 23:25:27.446 WARN returning error on read!
Sep 22 23:25:27.446 DEBG Read      :1454 deps:[JobId(1453)] res:false
Sep 22 23:25:27.446 WARN 1455 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:25:27.446 WARN returning error on read!
Sep 22 23:25:27.446 DEBG Read      :1454 deps:[JobId(1453)] res:false
Sep 22 23:25:27.446 INFO [lossy] skipping 1454
Sep 22 23:25:27.453 DEBG Read      :1454 deps:[JobId(1453)] res:true
Sep 22 23:25:27.474 ERRO [2] job id 1454 saw error GenericError("test error")
Sep 22 23:25:27.474 ERRO [2] job id 1454 saw error GenericError("test error")
Sep 22 23:25:27.476 DEBG Flush     :1455 extent_limit None deps:[JobId(1454), JobId(1453)] res:true f:188 g:1
Sep 22 23:25:27.852 DEBG [2] Read AckReady 1454, : downstairs
Sep 22 23:25:27.853 DEBG up_ds_listen was notified
Sep 22 23:25:27.853 DEBG up_ds_listen process 1454
Sep 22 23:25:27.853 DEBG [A] ack job 1454:455, : downstairs
Sep 22 23:25:27.906 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:27.907 DEBG IO Read  1456 has deps [JobId(1455)]
Sep 22 23:25:27.911 INFO [lossy] skipping 1456
Sep 22 23:25:27.912 WARN returning error on read!
Sep 22 23:25:27.912 DEBG Read      :1456 deps:[JobId(1455)] res:false
Sep 22 23:25:27.917 DEBG Read      :1456 deps:[JobId(1455)] res:true
Sep 22 23:25:27.939 ERRO [2] job id 1456 saw error GenericError("test error")
Sep 22 23:25:28.317 DEBG [2] Read AckReady 1456, : downstairs
Sep 22 23:25:28.318 DEBG up_ds_listen was notified
Sep 22 23:25:28.318 DEBG up_ds_listen process 1456
Sep 22 23:25:28.318 DEBG [A] ack job 1456:457, : downstairs
Sep 22 23:25:28.371 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:28.372 DEBG IO Flush 1457 has deps [JobId(1456), JobId(1455)]
Sep 22 23:25:28.372 DEBG IO Read  1458 has deps [JobId(1457)]
Sep 22 23:25:28.374 DEBG Flush     :1457 extent_limit None deps:[JobId(1456), JobId(1455)] res:true f:189 g:1
Sep 22 23:25:28.382 DEBG Read      :1458 deps:[JobId(1457)] res:true
Sep 22 23:25:28.780 DEBG [2] Read AckReady 1458, : downstairs
Sep 22 23:25:28.781 DEBG up_ds_listen was notified
Sep 22 23:25:28.781 DEBG up_ds_listen process 1458
Sep 22 23:25:28.781 DEBG [A] ack job 1458:459, : downstairs
Sep 22 23:25:28.833 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:28.835 DEBG IO Read  1459 has deps [JobId(1457)]
Sep 22 23:25:28.839 INFO [lossy] skipping 1459
Sep 22 23:25:28.845 DEBG Read      :1459 deps:[JobId(1457)] res:true
Sep 22 23:25:29.244 DEBG [2] Read AckReady 1459, : downstairs
Sep 22 23:25:29.245 DEBG up_ds_listen was notified
Sep 22 23:25:29.245 DEBG up_ds_listen process 1459
Sep 22 23:25:29.245 DEBG [A] ack job 1459:460, : downstairs
Sep 22 23:25:29.298 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:29.299 DEBG IO Flush 1460 has deps [JobId(1459), JobId(1458), JobId(1457)]
Sep 22 23:25:29.300 DEBG IO Read  1461 has deps [JobId(1460)]
Sep 22 23:25:29.302 INFO [lossy] skipping 1460
Sep 22 23:25:29.302 DEBG Flush     :1460 extent_limit None deps:[JobId(1459), JobId(1458), JobId(1457)] res:true f:190 g:1
Sep 22 23:25:29.304 INFO [lossy] sleeping 1 second
Sep 22 23:25:29.800 DEBG IO Flush 1462 has deps [JobId(1461), JobId(1460)]
Sep 22 23:25:30.312 DEBG Read      :1461 deps:[JobId(1460)] res:true
Sep 22 23:25:30.335 DEBG Flush     :1462 extent_limit None deps:[JobId(1461), JobId(1460)] res:true f:191 g:1
Sep 22 23:25:30.335 INFO [lossy] sleeping 1 second
Sep 22 23:25:30.711 DEBG [2] Read AckReady 1461, : downstairs
Sep 22 23:25:30.712 DEBG up_ds_listen was notified
Sep 22 23:25:30.712 DEBG up_ds_listen process 1461
Sep 22 23:25:30.712 DEBG [A] ack job 1461:462, : downstairs
Sep 22 23:25:30.765 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:30.766 DEBG IO Read  1463 has deps [JobId(1462)]
Sep 22 23:25:30.803 DEBG IO Flush 1464 has deps [JobId(1463), JobId(1462)]
Sep 22 23:25:31.342 DEBG Read      :1463 deps:[JobId(1462)] res:true
Sep 22 23:25:31.365 INFO [lossy] skipping 1464
Sep 22 23:25:31.365 DEBG Flush     :1464 extent_limit None deps:[JobId(1463), JobId(1462)] res:true f:192 g:1
Sep 22 23:25:31.741 DEBG [2] Read AckReady 1463, : downstairs
Sep 22 23:25:31.742 DEBG up_ds_listen was notified
Sep 22 23:25:31.742 DEBG up_ds_listen process 1463
Sep 22 23:25:31.743 DEBG [A] ack job 1463:464, : downstairs
Sep 22 23:25:31.795 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:31.797 DEBG IO Read  1465 has deps [JobId(1464)]
Sep 22 23:25:31.801 INFO [lossy] skipping 1465
Sep 22 23:25:31.801 WARN returning error on read!
Sep 22 23:25:31.801 DEBG Read      :1465 deps:[JobId(1464)] res:false
Sep 22 23:25:31.807 DEBG Read      :1465 deps:[JobId(1464)] res:true
Sep 22 23:25:31.828 DEBG IO Flush 1466 has deps [JobId(1465), JobId(1464)]
Sep 22 23:25:31.828 ERRO [2] job id 1465 saw error GenericError("test error")
Sep 22 23:25:31.830 INFO [lossy] sleeping 1 second
Sep 22 23:25:32.206 DEBG [2] Read AckReady 1465, : downstairs
Sep 22 23:25:32.207 DEBG up_ds_listen was notified
Sep 22 23:25:32.207 DEBG up_ds_listen process 1465
Sep 22 23:25:32.207 DEBG [A] ack job 1465:466, : downstairs
Sep 22 23:25:32.260 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:32.261 DEBG IO Read  1467 has deps [JobId(1466)]
Sep 22 23:25:32.330 DEBG IO Flush 1468 has deps [JobId(1467), JobId(1466)]
Sep 22 23:25:32.831 WARN returning error on flush!
Sep 22 23:25:32.831 DEBG Flush     :1466 extent_limit None deps:[JobId(1465), JobId(1464)] res:false f:193 g:1
Sep 22 23:25:32.831 DEBG Flush     :1466 extent_limit None deps:[JobId(1465), JobId(1464)] res:true f:193 g:1
Sep 22 23:25:32.831 INFO [lossy] sleeping 1 second
Sep 22 23:25:32.831 ERRO [2] job id 1466 saw error GenericError("test error")
Sep 22 23:25:33.832 INFO [lossy] skipping 1467
Sep 22 23:25:33.832 WARN 1468 job Flush for connection UpstairsConnection { upstairs_id: b937f86b-985c-4e08-8b98-07f7aae5f860, session_id: 27d81b06-bab5-4e4f-b4d3-4909cf4b2397, gen: 1 } waiting on 1 deps, role: work
Sep 22 23:25:33.839 DEBG Read      :1467 deps:[JobId(1466)] res:true
Sep 22 23:25:33.862 WARN returning error on flush!
Sep 22 23:25:33.862 DEBG Flush     :1468 extent_limit None deps:[JobId(1467), JobId(1466)] res:false f:194 g:1
Sep 22 23:25:33.862 INFO [lossy] skipping 1468
Sep 22 23:25:33.862 INFO [lossy] skipping 1468
Sep 22 23:25:33.862 WARN returning error on flush!
Sep 22 23:25:33.862 DEBG Flush     :1468 extent_limit None deps:[JobId(1467), JobId(1466)] res:false f:194 g:1
Sep 22 23:25:33.862 DEBG Flush     :1468 extent_limit None deps:[JobId(1467), JobId(1466)] res:true f:194 g:1
Sep 22 23:25:34.238 DEBG [2] Read AckReady 1467, : downstairs
Sep 22 23:25:34.239 ERRO [2] job id 1468 saw error GenericError("test error")
Sep 22 23:25:34.239 ERRO [2] job id 1468 saw error GenericError("test error")
Sep 22 23:25:34.239 DEBG up_ds_listen was notified
Sep 22 23:25:34.239 DEBG up_ds_listen process 1467
Sep 22 23:25:34.240 DEBG [A] ack job 1467:468, : downstairs
Sep 22 23:25:34.292 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:34.293 DEBG IO Read  1469 has deps [JobId(1468)]
Sep 22 23:25:34.297 INFO [lossy] skipping 1469
Sep 22 23:25:34.304 DEBG Read      :1469 deps:[JobId(1468)] res:true
Sep 22 23:25:34.704 DEBG [2] Read AckReady 1469, : downstairs
Sep 22 23:25:34.705 DEBG up_ds_listen was notified
Sep 22 23:25:34.705 DEBG up_ds_listen process 1469
Sep 22 23:25:34.705 DEBG [A] ack job 1469:470, : downstairs
Sep 22 23:25:34.758 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:34.759 DEBG IO Flush 1470 has deps [JobId(1469), JobId(1468)]
Sep 22 23:25:34.760 DEBG IO Read  1471 has deps [JobId(1470)]
Sep 22 23:25:34.762 DEBG Flush     :1470 extent_limit None deps:[JobId(1469), JobId(1468)] res:true f:195 g:1
Sep 22 23:25:34.770 DEBG Read      :1471 deps:[JobId(1470)] res:true
Sep 22 23:25:35.171 DEBG [2] Read AckReady 1471, : downstairs
Sep 22 23:25:35.172 DEBG up_ds_listen was notified
Sep 22 23:25:35.172 DEBG up_ds_listen process 1471
Sep 22 23:25:35.172 DEBG [A] ack job 1471:472, : downstairs
Sep 22 23:25:35.225 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:35.226 DEBG IO Read  1472 has deps [JobId(1470)]
Sep 22 23:25:35.231 WARN returning error on read!
Sep 22 23:25:35.231 DEBG Read      :1472 deps:[JobId(1470)] res:false
Sep 22 23:25:35.231 INFO [lossy] skipping 1472
Sep 22 23:25:35.236 DEBG Read      :1472 deps:[JobId(1470)] res:true
Sep 22 23:25:35.258 ERRO [2] job id 1472 saw error GenericError("test error")
Sep 22 23:25:35.636 DEBG [2] Read AckReady 1472, : downstairs
Sep 22 23:25:35.637 DEBG up_ds_listen was notified
Sep 22 23:25:35.637 DEBG up_ds_listen process 1472
Sep 22 23:25:35.637 DEBG [A] ack job 1472:473, : downstairs
Sep 22 23:25:35.690 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:35.691 DEBG IO Flush 1473 has deps [JobId(1472), JobId(1471), JobId(1470)]
Sep 22 23:25:35.692 DEBG IO Read  1474 has deps [JobId(1473)]
Sep 22 23:25:35.694 DEBG Flush     :1473 extent_limit None deps:[JobId(1472), JobId(1471), JobId(1470)] res:true f:196 g:1
Sep 22 23:25:35.696 INFO [lossy] sleeping 1 second
Sep 22 23:25:36.192 DEBG IO Flush 1475 has deps [JobId(1474), JobId(1473)]
Sep 22 23:25:36.703 DEBG Read      :1474 deps:[JobId(1473)] res:true
Sep 22 23:25:36.726 DEBG Flush     :1475 extent_limit None deps:[JobId(1474), JobId(1473)] res:true f:197 g:1
Sep 22 23:25:36.726 INFO [lossy] sleeping 1 second
Sep 22 23:25:37.103 DEBG [2] Read AckReady 1474, : downstairs
Sep 22 23:25:37.104 DEBG up_ds_listen was notified
Sep 22 23:25:37.104 DEBG up_ds_listen process 1474
Sep 22 23:25:37.104 DEBG [A] ack job 1474:475, : downstairs
Sep 22 23:25:37.157 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:37.158 DEBG IO Read  1476 has deps [JobId(1475)]
Sep 22 23:25:37.195 DEBG IO Flush 1477 has deps [JobId(1476), JobId(1475)]
Sep 22 23:25:37.728 INFO [lossy] skipping 1476
Sep 22 23:25:37.728 WARN returning error on read!
Sep 22 23:25:37.728 DEBG Read      :1476 deps:[JobId(1475)] res:false
Sep 22 23:25:37.735 DEBG Read      :1476 deps:[JobId(1475)] res:true
Sep 22 23:25:37.756 ERRO [2] job id 1476 saw error GenericError("test error")
Sep 22 23:25:37.757 INFO [lossy] skipping 1477
Sep 22 23:25:37.758 DEBG Flush     :1477 extent_limit None deps:[JobId(1476), JobId(1475)] res:true f:198 g:1
Sep 22 23:25:37.758 INFO [lossy] sleeping 1 second
Sep 22 23:25:38.134 DEBG [2] Read AckReady 1476, : downstairs
Sep 22 23:25:38.135 DEBG up_ds_listen was notified
Sep 22 23:25:38.135 DEBG up_ds_listen process 1476
Sep 22 23:25:38.136 DEBG [A] ack job 1476:477, : downstairs
Sep 22 23:25:38.188 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:38.190 DEBG IO Read  1478 has deps [JobId(1477)]
Sep 22 23:25:38.197 DEBG IO Flush 1479 has deps [JobId(1478), JobId(1477)]
Sep 22 23:25:38.758 INFO [lossy] skipping 1478
Sep 22 23:25:38.764 DEBG Read      :1478 deps:[JobId(1477)] res:true
Sep 22 23:25:38.787 INFO [lossy] skipping 1479
Sep 22 23:25:38.787 INFO [lossy] skipping 1479
Sep 22 23:25:38.787 DEBG Flush     :1479 extent_limit None deps:[JobId(1478), JobId(1477)] res:true f:199 g:1
Sep 22 23:25:38.787 INFO [lossy] sleeping 1 second
Sep 22 23:25:39.164 DEBG [2] Read AckReady 1478, : downstairs
Sep 22 23:25:39.165 DEBG up_ds_listen was notified
Sep 22 23:25:39.165 DEBG up_ds_listen process 1478
Sep 22 23:25:39.165 DEBG [A] ack job 1478:479, : downstairs
Sep 22 23:25:39.218 DEBG up_ds_listen checked 1 jobs, back to waiting
Sep 22 23:25:39.219 DEBG IO Read  1480 has deps [JobId(1479)]
Sep 22 23:25:39.720 DEBG IO Flush 1481 has deps [JobId(1480), JobId(1479)]
Sep 22 23:25:39.795 DEBG Read      :1480 deps:[JobId(1479)] res:true
Sep 22 23:25:39.818 DEBG Flush     :1481 extent_limit None deps:[JobId(1480), JobId(1479)] res:true f:200 g:1
Sep 22 23:25:40.195 DEBG [2] Read AckReady 1480, : downstairs
Sep 22 23:25:40.197 DEBG up_ds_listen was notified
Sep 22 23:25:40.197 DEBG up_ds_listen process 1480
Sep 22 23:25:40.197 DEBG [A] ack job 1480:481, : downstairs
Sep 22 23:25:40.249 DEBG up_ds_listen checked 1 jobs, back to waiting
test test::integration_test_problematic_downstairs ... ok

test result: ok. 57 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 242.88s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_nbd_server-8c1612631a1669fd --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_package-e8ff0170d25e0da5 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_pantry-8e1bf492bfe90e8c --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_pantry-b51bc30f7a0cbfa5 --nocapture`

running 1 test
test tests::test_crucible_pantry_openapi ... ok

test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_pantry_client-8a27f01eb086219e --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_protocol-c776b78ce4b42bf6 --nocapture`

running 9 tests
test tests::latest_message_version ... ok
test tests::correctly_detect_truncated_message ... ok
test tests::rt_evp ... ok
test tests::rt_ev_0 ... ok
test tests::rt_ev_7 ... ok
test tests::rt_imok ... ok
test tests::rt_here_i_am ... ok
test tests::rt_ruok ... ok
test tests::rt_yes_its_me ... ok

test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_protocol-d81e6562be2ffe77 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crucible_smf-6114df38a9482a0c --nocapture`

running 10 tests
SCF_LIMIT_MAX_NAME_LENGTH = 119
SCF_LIMIT_MAX_VALUE_LENGTH = 4095
SCF_LIMIT_MAX_PG_TYPE_LENGTH = 119
SCF_LIMIT_MAX_FMRI_LENGTH = 628
test scf_sys::tests::limits ... ok
test scf_sys::tests::handle ... ok
name = Ok("localhost")
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
test tests::scope_not_set ... ok
test tests::iter ... ok
test tests::basic ... ok
milestone/multi-user
test tests::scope_local ... ok
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
test tests::scope_iter ... ok
milestone/multi-user
milestone/multi-user
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
milestone/name-services
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
milestone/single-user
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
milestone/name-services
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
        initial
network/datalink-management
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
system/install-discovery
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
        previous
system/device/local
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
milestone/single-user
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
network/physical
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
        start
network/initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/ip-interface-management
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/datalink-management
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/loopback
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/iptun
milestone/name-services
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/netcfg
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/install-discovery
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
network/rpc/bind
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/boot-archive
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        initial
system/device/local
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
milestone/devices
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/filesystem/local
        previous
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/filesystem/minimal
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        running
network/physical
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/filesystem/root
        start
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/filesystem/usr
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    nwam
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/identity
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/manifest-import
network/initial
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
milestone/single-user
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/svc/global
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
system/svc/restarter
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
milestone/multi-user-server
network/ip-interface-management
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/inetd-upgrade
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        last-import
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/console-login
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/loopback
system/utmp
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
application/management/net-snmp
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        start
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
smf/manifest
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/iptun
application/pkg/repositories-setup
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
application/pkg/dynamic-mirror
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/datalink-management
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
application/pkg/mirror
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/netcfg
    default
application/pkg/server
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
application/security/tcsd
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
milestone/sysconfig
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/rpc/bind
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
milestone/network
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
network/ntp
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/ipmp
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        start
system/boot-archive
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/routing/rdisc
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
network/routing/route
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/routing/ndp
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
milestone/devices
system/install-discovery
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/routing/legacy-routing
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
network/routing/ripng
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/inetd
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/device/local
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/filesystem/local
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/bridge
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
    default
network/ipv4-forwarding
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
network/ipv6-forwarding
        initial
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/filesystem/minimal
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
network/ipqos
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
network/ipsec/ipsecalgs
        previous
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
network/ipsec/policy
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        running
system/filesystem/root
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/ipsec/manual-key
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/ipsec/ike
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/install
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/filesystem/usr
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/shares/group
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
network/physical
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/filesystem/reparse
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
network/rpc/smserver
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/identity
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/rpc/keyserv
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        initial
    domain
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/rpc/gss
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    node
        last-import
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/ipfilter
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/ldap/client
system/manifest-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/smb/client
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        start
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/smb/server
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/routing-setup
system/svc/global
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    nwam
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
network/npiv_config
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/device/fc-fabric
        initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/svc/restarter
network/ssh
        last-import
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
network/varpd
        previous
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
network/security/kadmin
        running
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/security/krb5kdc
milestone/multi-user-server
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/security/ktkt_warn
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/device-discovery/printers
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/service
network/inetd-upgrade
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
network/nis/client
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/location
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/dns/client
system/console-login
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/name-service-cache
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        previous
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
network/nfs/mapid
    vt2
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    vt3
network/chrony
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        start
    vt4
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/ibd-post-upgrade
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    vt5
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/tftp/udp6
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    vt6
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/netmask
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/ip-interface-management
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/dns/multicast
system/utmp
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
network/dns/install
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/nfs/log
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
        initial
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/nfs/rquota
application/management/net-snmp
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
network/nfs/client
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/nfs/server
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/nfs/cbd
smf/manifest
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
        start
network/nfs/status
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/nfs/nlockmgr
application/pkg/repositories-setup
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
platform/i86pc/acpihpd
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/sac
network/loopback
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/fcoe_initiator
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
application/pkg/dynamic-mirror
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/fmd
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/sysevent
        initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/boot-config
        last-import
application/pkg/mirror
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/picl
        previous
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/coreadm
        running
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/hal
        start
application/pkg/server
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/resource-mgmt
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/rcap
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/system-log
application/security/tcsd
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/iptun
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/dumpadm
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/dbus
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
milestone/sysconfig
system/pools
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
system/power
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/keymap
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
milestone/network
system/auditset
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
system/stmf
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/hotplug
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/ntp
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/rbac
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
system/logadm-upgrade
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/netcfg
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/hostid
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
network/ipmp
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
system/filesystem/autofs
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
system/cron
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/illumos/userscript
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        last-import
network/routing/rdisc
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/vtdaemon
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/boot-archive-update
        running
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/cryptosvc
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        start
network/routing/route
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/intrd
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/rmtmpfiles
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/t6init
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/rpc/bind
network/routing/ndp
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/auditd
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
system/idmap
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/zones-monitoring
network/routing/legacy-routing
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/zones
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        last-import
    ipv4
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/ipcc
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        previous
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    ipv6
system/update-man-index
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/process-security
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        start
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/routing/ripng
system/fm/notify-params
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
system/pkgserv
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/extended-accounting
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/boot-archive
system/consadm
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/inetd
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/scheduler
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    default
system/pfexec
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/illumos/metadata
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/sar
network/bridge
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/early-manifest-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/fcoe_target
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        start
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
network/ipv4-forwarding
system/device/mpxio-upgrade
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
system/device/audio
    default
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
system/device/allocate
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
milestone/devices
smf/legacy_run
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/ipv6-forwarding
Ok(Service { scf: Scf { handle: 0x6b3050 }, service: 0x6b4810 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
site/buildomat/agent
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
test tests::service_iter ... Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
ok
        last-import
network/ipqos
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/ipsec/ipsecalgs
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/filesystem/local
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/ipsec/policy
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
network/ipsec/manual-key
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/ipsec/ike
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/filesystem/minimal
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/install
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/shares/group
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        start
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    zfs
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/filesystem/reparse
system/filesystem/root
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/rpc/smserver
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        start
network/rpc/keyserv
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/filesystem/usr
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/rpc/gss
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        last-import
network/ipfilter
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/ldap/client
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/identity
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/smb/client
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    domain
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/smb/server
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        running
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/routing-setup
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    node
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        last-import
network/npiv_config
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/device/fc-fabric
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/manifest-import
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
network/ssh
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
network/varpd
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/security/kadmin
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
    default
system/svc/global
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/security/krb5kdc
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/security/ktkt_warn
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/device-discovery/printers
system/svc/restarter
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    snmp
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/service
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        previous
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/nis/client
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
milestone/multi-user-server
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    default
network/location
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        initial
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        previous
network/dns/client
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/name-service-cache
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
network/inetd-upgrade
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/nfs/mapid
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        running
network/chrony
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/ibd-post-upgrade
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/console-login
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        initial
network/tftp/udp6
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/netmask
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    vt2
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/dns/multicast
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        last-import
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/dns/install
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    vt3
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/nfs/log
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/nfs/rquota
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    vt4
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        last-import
network/nfs/client
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/nfs/server
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    vt5
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/nfs/cbd
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        running
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    vt6
network/nfs/status
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        initial
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/nfs/nlockmgr
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
platform/i86pc/acpihpd
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/utmp
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/sac
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/fcoe_initiator
        start
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
application/management/net-snmp
system/fmd
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/sysevent
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/boot-config
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
smf/manifest
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
application/pkg/repositories-setup
system/picl
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/coreadm
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/hal
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
application/pkg/dynamic-mirror
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        initial
system/resource-mgmt
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/rcap
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
application/pkg/mirror
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/system-log
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    rsyslog
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/dumpadm
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
application/pkg/server
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/dbus
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/pools
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
application/security/tcsd
system/power
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/keymap
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/auditset
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
milestone/sysconfig
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/stmf
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/hotplug
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
milestone/network
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/rbac
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/logadm-upgrade
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/hostid
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ntp
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/filesystem/autofs
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
        last-import
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        start
system/cron
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ipmp
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/illumos/userscript
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/vtdaemon
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/boot-archive-update
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
network/routing/rdisc
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/cryptosvc
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/intrd
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/routing/route
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
system/rmtmpfiles
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        running
system/t6init
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/routing/ndp
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/auditd
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        last-import
system/idmap
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/zones-monitoring
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
    default
network/routing/legacy-routing
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    ipv4
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/zones
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/ipcc
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    ipv6
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/update-man-index
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/process-security
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/routing/ripng
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
system/fm/notify-params
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/pkgserv
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
    default
network/inetd
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/extended-accounting
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    flow
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    net
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    process
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    task
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/consadm
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/bridge
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
network/ipv4-forwarding
system/scheduler
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        last-import
system/pfexec
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/illumos/metadata
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ipv6-forwarding
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        initial
system/sar
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/early-manifest-import
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
    default
network/ipqos
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/fcoe_target
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
system/device/mpxio-upgrade
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ipsec/ipsecalgs
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
system/device/audio
    default
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
    default
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
        running
system/device/allocate
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
smf/legacy_run
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ipsec/policy
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ipsec/manual-key
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x68c050 }, service: 0x6b4950 })
site/buildomat/agent
    Ok(Instance { scf: Scf { handle: 0x68c050 }, instance: 0x6b4710 })
    default
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ipsec/ike
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
test tests::instance_iter ... ok
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/install
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/shares/group
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    zfs
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/filesystem/reparse
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/rpc/smserver
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/rpc/keyserv
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/rpc/gss
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ipfilter
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ldap/client
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/smb/client
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/smb/server
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/routing-setup
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/npiv_config
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/device/fc-fabric
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ssh
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/varpd
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/security/kadmin
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/security/krb5kdc
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/security/ktkt_warn
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/device-discovery/printers
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    snmp
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/service
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/nis/client
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/location
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/dns/client
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/name-service-cache
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/nfs/mapid
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/chrony
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/ibd-post-upgrade
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/tftp/udp6
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/netmask
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/dns/multicast
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/dns/install
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/nfs/log
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/nfs/rquota
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/nfs/client
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/nfs/server
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/nfs/cbd
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/nfs/status
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
network/nfs/nlockmgr
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
platform/i86pc/acpihpd
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/sac
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/fcoe_initiator
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/fmd
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/sysevent
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/boot-config
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/picl
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/coreadm
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/hal
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/resource-mgmt
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/rcap
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/system-log
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    rsyslog
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/dumpadm
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/dbus
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/pools
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/power
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/keymap
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/auditset
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/stmf
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/hotplug
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/rbac
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/logadm-upgrade
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/hostid
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/filesystem/autofs
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/cron
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/illumos/userscript
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/vtdaemon
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/boot-archive-update
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/cryptosvc
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/intrd
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/rmtmpfiles
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/t6init
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/auditd
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/idmap
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/zones-monitoring
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/zones
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        previous
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/ipcc
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/update-man-index
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/process-security
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/fm/notify-params
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/pkgserv
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/extended-accounting
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    flow
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    net
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    process
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    task
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/consadm
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/scheduler
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/pfexec
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/illumos/metadata
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/sar
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/early-manifest-import
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/fcoe_target
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/device/mpxio-upgrade
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/device/audio
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
system/device/allocate
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
smf/legacy_run
Ok(Service { scf: Scf { handle: 0x6b7a90 }, service: 0x6b47d0 })
site/buildomat/agent
    Ok(Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 })
    default
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        last-import
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        running
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        initial
    Instance { scf: Scf { handle: 0x6b7a90 }, instance: 0x6b46d0 }
        start
test tests::snapshot_iter ... ok

test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.11s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crudd-3e9b00990c25260e --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/crutest-af78e92d646e2d06 --nocapture`

running 64 tests
test protocol::tests::correctly_detect_truncated_message ... ok
test protocol::tests::rt_commit ... ok
test protocol::tests::rt_deactivate ... ok
test protocol::tests::rt_activate ... ok
test protocol::tests::rt_done_ok ... ok
test protocol::tests::rt_generic ... ok
test protocol::tests::rt_info_please ... ok
test protocol::tests::rt_info ... ok
test protocol::tests::rt_is_active ... ok
test protocol::tests::rt_is_show ... ok
test protocol::tests::rt_my_uuid ... ok
test protocol::tests::rt_perf ... ok
test protocol::tests::rt_read ... ok
test protocol::tests::rt_uuid ... ok
test protocol::tests::rt_replace ... ok
test protocol::tests::rt_write ... ok
test protocol::tests::rt_write_unwritten ... ok
test test::test_95_2 ... ok
test test::test_95_20 ... ok
test test::test_95_21 ... ok
test test::test_95_10 ... ok
test test::test_perc_bad_big_perc ... ok
test test::test_perc_bad_perc ... ok
test test::test_95_small ... ok
test test::test_perc_mixed ... ok
test test::test_read_compare_empty ... ok
Mismatch     Block::0 bo:1 Volume offset:1  Expected:1 Got:2
test test::test_read_compare_1 ... ok
Mismatch     Block::0 bo:2 Volume offset:2  Expected:1 Got:2
Mismatch     Block::0 bo:3 Volume offset:3  Expected:1 Got:2
Mismatch     Block::0 bo:1 Volume offset:1  Expected:3 Got:2
test test::test_read_compare_commit ... okMismatch     Block::0 bo:2 Volume offset:2  Expected:3 Got:2
Mismatch     Block::0 bo:4 Volume offset:4  Expected:1 Got:2

Mismatch     Block::0 bo:3 Volume offset:3  Expected:3 Got:2
Mismatch     Block::0 bo:5 Volume offset:5  Expected:1 Got:2
Mismatch     Block::0 bo:4 Volume offset:4  Expected:3 Got:2
Mismatch     Block::0 bo:6 Volume offset:6  Expected:1 Got:2
Mismatch     Block::0 bo:5 Volume offset:5  Expected:3 Got:2
Mismatch     Block::0 bo:7 Volume offset:7  Expected:1 Got:2
Mismatch     Block::0 bo:6 Volume offset:6  Expected:3 Got:2
Mismatch     Block::0 bo:8 Volume offset:8  Expected:1 Got:2
Mismatch     Block::0 bo:7 Volume offset:7  Expected:3 Got:2
Mismatch     Block::0 bo:9 Volume offset:9  Expected:1 Got:2
Mismatch     Block::0 bo:8 Volume offset:8  Expected:3 Got:2
Mismatch     Block::0 bo:10 Volume offset:10  Expected:1 Got:2
Mismatch     Block::0 bo:9 Volume offset:9  Expected:3 Got:2
Mismatch     Block::0 bo:11 Volume offset:11  Expected:1 Got:2
Mismatch     Block::0 bo:10 Volume offset:10  Expected:3 Got:2
Mismatch     Block::0 bo:12 Volume offset:12  Expected:1 Got:2
test test::test_read_compare ... Mismatch     Block::0 bo:11 Volume offset:11  Expected:3 Got:2
Mismatch     Block::0 bo:12 Volume offset:12  Expected:3 Got:2
ok
Mismatch     Block::0 bo:13 Volume offset:13  Expected:3 Got:2
Mismatch     Block::0 bo:13 Volume offset:13  Expected:1 Got:2
Mismatch     Block::0 bo:14 Volume offset:14  Expected:3 Got:2
Mismatch     Block::0 bo:14 Volume offset:14  Expected:1 Got:2
Mismatch     Block::0 bo:15 Volume offset:15  Expected:3 Got:2
Mismatch     Block::0 bo:15 Volume offset:15  Expected:1 Got:2
Mismatch     Block::0 bo:16 Volume offset:16  Expected:3 Got:2
Mismatch     Block::0 bo:16 Volume offset:16  Expected:1 Got:2
Mismatch     Block::0 bo:17 Volume offset:17  Expected:3 Got:2
Mismatch     Block::0 bo:17 Volume offset:17  Expected:1 Got:2
Mismatch     Block::0 bo:18 Volume offset:18  Expected:3 Got:2
Mismatch     Block::0 bo:18 Volume offset:18  Expected:1 Got:2
Mismatch     Block::0 bo:19 Volume offset:19  Expected:3 Got:2
Mismatch     Block::0 bo:19 Volume offset:19  Expected:1 Got:2
Mismatch     Block::0 bo:20 Volume offset:20  Expected:3 Got:2
Mismatch     Block::0 bo:20 Volume offset:20  Expected:1 Got:2
test test::test_read_compare_empty_data ... ok
Mismatch     Block::0 bo:21 Volume offset:21  Expected:1 Got:2
Mismatch     Block::0 bo:22 Volume offset:22  Expected:1 Got:2
Mismatch     Block::0 bo:23 Volume offset:23  Expected:1 Got:2
Mismatch     Block::0 bo:24 Volume offset:24  Expected:1 Got:2
Mismatch     Block::0 bo:21 Volume offset:21  Expected:3 Got:2
Mismatch     Block::0 bo:25 Volume offset:25  Expected:1 Got:2
Mismatch     Block::0 bo:22 Volume offset:22  Expected:3 Got:2
Mismatch     Block::2 bo:1 Volume offset:1025  Expected:2 Got:9
Mismatch     Block::0 bo:23 Volume offset:23  Expected:3 Got:2
Mismatch     Block::0 bo:24 Volume offset:24  Expected:3 Got:2
Mismatch     Block::0 bo:25 Volume offset:25  Expected:3 Got:2
Mismatch     Block::0 bo:26 Volume offset:26  Expected:3 Got:2
Mismatch     Block::0 bo:27 Volume offset:27  Expected:3 Got:2
Mismatch     Block::0 bo:28 Volume offset:28  Expected:3 Got:2
Mismatch     Block::0 bo:29 Volume offset:29  Expected:3 Got:2
SPEC  v:2  min_av:254 cur_av:1  cm:1022 cc:1025
Mismatch     Block::0 bo:30 Volume offset:30  Expected:3 Got:2
SPEC  v:254  min_av:254 cur_av:1  cm:1022 cc:1025
Mismatch     Block::0 bo:31 Volume offset:31  Expected:3 Got:2
new cur is 1022 from min
Mismatch     Block::0 bo:32 Volume offset:32  Expected:3 Got:2
SPEC  v:255  min_av:254 cur_av:1  cm:1022 cc:1025
Mismatch     Block::0 bo:33 Volume offset:33  Expected:3 Got:2
new cur is 1023 from min
Mismatch     Block::0 bo:34 Volume offset:34  Expected:3 Got:2
SPEC  v:0  min_av:254 cur_av:1  cm:1022 cc:1025
Mismatch     Block::0 bo:35 Volume offset:35  Expected:3 Got:2
new cur is 1024 from cur
Mismatch     Block::0 bo:36 Volume offset:36  Expected:3 Got:2
SPEC  v:1  min_av:254 cur_av:1  cm:1022 cc:1025
test test::test_read_compare_span ... Mismatch     Block::0 bo:37 Volume offset:37  Expected:3 Got:2
new cur is 1025 from cur
ok
Mismatch     Block::0 bo:38 Volume offset:38  Expected:3 Got:2
Mismatch     Block::0 bo:39 Volume offset:39  Expected:3 Got:2
Mismatch     Block::0 bo:40 Volume offset:40  Expected:3 Got:2
Mismatch     Block::0 bo:41 Volume offset:41  Expected:3 Got:2
Mismatch     Block::0 bo:42 Volume offset:42  Expected:3 Got:2
test test::test_read_compare_span_fail ... Mismatch     Block::0 bo:43 Volume offset:43  Expected:3 Got:2
okMismatch     Block::0 bo:44 Volume offset:44  Expected:3 Got:2
Mismatch     Block::0 bo:45 Volume offset:45  Expected:3 Got:2
Mismatch     Block::0 bo:46 Volume offset:46  Expected:3 Got:2
Mismatch     Block::0 bo:47 Volume offset:47  Expected:3 Got:2
Mismatch     Block::0 bo:48 Volume offset:48  Expected:3 Got:2
Mismatch     Block::0 bo:49 Volume offset:49  Expected:3 Got:2
Mismatch     Block::0 bo:50 Volume offset:50  Expected:3 Got:2
Mismatch     Block::0 bo:51 Volume offset:51  Expected:3 Got:2
Mismatch     Block::0 bo:52 Volume offset:52  Expected:3 Got:2
Mismatch     Block::0 bo:53 Volume offset:53  Expected:3 Got:2
Mismatch     Block::0 bo:54 Volume offset:54  Expected:3 Got:2
Mismatch     Block::0 bo:55 Volume offset:55  Expected:3 Got:2
Mismatch     Block::0 bo:56 Volume offset:56  Expected:3 Got:2
Mismatch     Block::0 bo:57 Volume offset:57  Expected:3 Got:2
Mismatch     Block::0 bo:58 Volume offset:58  Expected:3 Got:2
Mismatch     Block::0 bo:59 Volume offset:59  Expected:3 Got:2
Mismatch     Block::0 bo:60 Volume offset:60  Expected:3 Got:2
Mismatch     Block::0 bo:61 Volume offset:61  Expected:3 Got:2
Mismatch     Block::0 bo:62 Volume offset:62  Expected:3 Got:2

Mismatch     Block::0 bo:63 Volume offset:63  Expected:3 Got:2
Mismatch     Block::0 bo:64 Volume offset:64  Expected:3 Got:2
Mismatch     Block::0 bo:65 Volume offset:65  Expected:3 Got:2
Mismatch     Block::0 bo:66 Volume offset:66  Expected:3 Got:2
Mismatch     Block::0 bo:67 Volume offset:67  Expected:3 Got:2
Mismatch     Block::0 bo:68 Volume offset:68  Expected:3 Got:2
Mismatch     Block::0 bo:69 Volume offset:69  Expected:3 Got:2
Mismatch     Block::0 bo:70 Volume offset:70  Expected:3 Got:2
Mismatch     Block::0 bo:71 Volume offset:71  Expected:3 Got:2
test test::test_wl_commit_1024_range_no_update_above_rollover ... ok
Mismatch     Block::0 bo:72 Volume offset:72  Expected:3 Got:2
SPEC  v:253  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:252  min_av:253 cur_av:0  cm:1021 cc:1024
SPEC  v:253  min_av:253 cur_av:0  cm:1021 cc:1024
SPEC  v:254  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:254  min_av:255 cur_av:1  cm:1023 cc:1025
SPEC  v:255  min_av:255 cur_av:1  cm:1023 cc:1025
new cur is 1023 from min
SPEC  v:0  min_av:255 cur_av:1  cm:1023 cc:1025
new cur is 1024 from cur
SPEC  v:1  min_av:255 cur_av:1  cm:1023 cc:1025
new cur is 1025 from cur
SPEC  v:2  min_av:255 cur_av:1  cm:1023 cc:1025
Mismatch     Block::2 bo:2 Volume offset:1026  Expected:2 Got:9
new cur is 1021 from min
SPEC  v:254  min_av:253 cur_av:0  cm:1021 cc:1024
new cur is 1022 from min
Mismatch     Block::0 bo:73 Volume offset:73  Expected:3 Got:2
SPEC  v:255  min_av:253 cur_av:0  cm:1021 cc:1024
new cur is 1023 from min
SPEC  v:0  min_av:253 cur_av:0  cm:1021 cc:1024
Mismatch     Block::0 bo:74 Volume offset:74  Expected:3 Got:2
test test::test_wl_commit_1024_range_rollover_min_at ... Mismatch     Block::0 bo:75 Volume offset:75  Expected:3 Got:2
Mismatch     Block::0 bo:76 Volume offset:76  Expected:3 Got:2
Mismatch     Block::0 bo:77 Volume offset:77  Expected:3 Got:2
Mismatch     Block::0 bo:78 Volume offset:78  Expected:3 Got:2
Mismatch     Block::0 bo:79 Volume offset:79  Expected:3 Got:2
Mismatch     Block::0 bo:80 Volume offset:80  Expected:3 Got:2
Mismatch     Block::0 bo:81 Volume offset:81  Expected:3 Got:2
Mismatch     Block::0 bo:82 Volume offset:82  Expected:3 Got:2
Mismatch     Block::0 bo:83 Volume offset:83  Expected:3 Got:2
Mismatch     Block::0 bo:84 Volume offset:84  Expected:3 Got:2
Mismatch     Block::0 bo:85 Volume offset:85  Expected:3 Got:2
Mismatch     Block::0 bo:86 Volume offset:86  Expected:3 Got:2
new cur is 1024 from cur
Mismatch     Block::0 bo:26 Volume offset:26  Expected:1 Got:2
Mismatch     Block::0 bo:27 Volume offset:27  Expected:1 Got:2
Mismatch     Block::0 bo:28 Volume offset:28  Expected:1 Got:2
Mismatch     Block::0 bo:29 Volume offset:29  Expected:1 Got:2
Mismatch     Block::0 bo:30 Volume offset:30  Expected:1 Got:2
Mismatch     Block::0 bo:31 Volume offset:31  Expected:1 Got:2
Mismatch     Block::0 bo:32 Volume offset:32  Expected:1 Got:2
Mismatch     Block::0 bo:33 Volume offset:33  Expected:1 Got:2
Mismatch     Block::0 bo:34 Volume offset:34  Expected:1 Got:2
Mismatch     Block::0 bo:35 Volume offset:35  Expected:1 Got:2
Mismatch     Block::0 bo:36 Volume offset:36  Expected:1 Got:2
Mismatch     Block::0 bo:37 Volume offset:37  Expected:1 Got:2
Mismatch     Block::0 bo:38 Volume offset:38  Expected:1 Got:2
Mismatch     Block::0 bo:39 Volume offset:39  Expected:1 Got:2
okMismatch     Block::0 bo:40 Volume offset:40  Expected:1 Got:2

Mismatch     Block::0 bo:41 Volume offset:41  Expected:1 Got:2
SPEC  v:1  min_av:253 cur_av:0  cm:1021 cc:1024
Mismatch     Block::0 bo:87 Volume offset:87  Expected:3 Got:2
Mismatch     Block::0 bo:88 Volume offset:88  Expected:3 Got:2
Mismatch     Block::0 bo:89 Volume offset:89  Expected:3 Got:2
Mismatch     Block::0 bo:90 Volume offset:90  Expected:3 Got:2
Mismatch     Block::0 bo:91 Volume offset:91  Expected:3 Got:2
Mismatch     Block::0 bo:92 Volume offset:92  Expected:3 Got:2
Mismatch     Block::0 bo:93 Volume offset:93  Expected:3 Got:2
Mismatch     Block::0 bo:94 Volume offset:94  Expected:3 Got:2
Mismatch     Block::0 bo:95 Volume offset:95  Expected:3 Got:2
Mismatch     Block::0 bo:96 Volume offset:96  Expected:3 Got:2
Mismatch     Block::0 bo:97 Volume offset:97  Expected:3 Got:2
Mismatch     Block::0 bo:98 Volume offset:98  Expected:3 Got:2
Mismatch     Block::0 bo:99 Volume offset:99  Expected:3 Got:2
Mismatch     Block::0 bo:100 Volume offset:100  Expected:3 Got:2
Mismatch     Block::0 bo:42 Volume offset:42  Expected:1 Got:2
Mismatch     Block::0 bo:101 Volume offset:101  Expected:3 Got:2
Mismatch     Block::0 bo:43 Volume offset:43  Expected:1 Got:2
Mismatch     Block::0 bo:102 Volume offset:102  Expected:3 Got:2
Mismatch     Block::0 bo:44 Volume offset:44  Expected:1 Got:2
Mismatch     Block::0 bo:103 Volume offset:103  Expected:3 Got:2
Mismatch     Block::0 bo:45 Volume offset:45  Expected:1 Got:2
Mismatch     Block::0 bo:104 Volume offset:104  Expected:3 Got:2
Mismatch     Block::0 bo:46 Volume offset:46  Expected:1 Got:2
Mismatch     Block::0 bo:105 Volume offset:105  Expected:3 Got:2
Mismatch     Block::0 bo:47 Volume offset:47  Expected:1 Got:2
Mismatch     Block::0 bo:106 Volume offset:106  Expected:3 Got:2
Mismatch     Block::0 bo:48 Volume offset:48  Expected:1 Got:2
Mismatch     Block::0 bo:107 Volume offset:107  Expected:3 Got:2
Mismatch     Block::0 bo:49 Volume offset:49  Expected:1 Got:2
Mismatch     Block::0 bo:108 Volume offset:108  Expected:3 Got:2
Mismatch     Block::0 bo:50 Volume offset:50  Expected:1 Got:2
Mismatch     Block::0 bo:109 Volume offset:109  Expected:3 Got:2
Mismatch     Block::0 bo:51 Volume offset:51  Expected:1 Got:2
Mismatch     Block::0 bo:110 Volume offset:110  Expected:3 Got:2
Mismatch     Block::0 bo:52 Volume offset:52  Expected:1 Got:2
Mismatch     Block::0 bo:111 Volume offset:111  Expected:3 Got:2
Mismatch     Block::0 bo:53 Volume offset:53  Expected:1 Got:2
Mismatch     Block::0 bo:112 Volume offset:112  Expected:3 Got:2
Mismatch     Block::0 bo:54 Volume offset:54  Expected:1 Got:2
Mismatch     Block::0 bo:113 Volume offset:113  Expected:3 Got:2
Mismatch     Block::0 bo:55 Volume offset:55  Expected:1 Got:2
Mismatch     Block::0 bo:114 Volume offset:114  Expected:3 Got:2
Mismatch     Block::0 bo:56 Volume offset:56  Expected:1 Got:2
Mismatch     Block::0 bo:115 Volume offset:115  Expected:3 Got:2
Mismatch     Block::0 bo:57 Volume offset:57  Expected:1 Got:2
Mismatch     Block::0 bo:116 Volume offset:116  Expected:3 Got:2
Mismatch     Block::0 bo:58 Volume offset:58  Expected:1 Got:2
test test::test_read_compare_span_fail_2 ... Mismatch     Block::0 bo:117 Volume offset:117  Expected:3 Got:2
ok
Mismatch     Block::0 bo:118 Volume offset:118  Expected:3 Got:2
Mismatch     Block::0 bo:59 Volume offset:59  Expected:1 Got:2
Mismatch     Block::0 bo:119 Volume offset:119  Expected:3 Got:2
Mismatch     Block::0 bo:60 Volume offset:60  Expected:1 Got:2
Mismatch     Block::0 bo:120 Volume offset:120  Expected:3 Got:2
SPEC  v:253  min_av:254 cur_av:1  cm:1022 cc:1025
test test::test_wl_commit_1024_range_rollover_max_at ... SPEC  v:254  min_av:254 cur_av:1  cm:1022 cc:1025
oknew cur is 1022 from min
SPEC  v:255  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:253  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1023 from min
Mismatch     Block::0 bo:121 Volume offset:121  Expected:3 Got:2
SPEC  v:0  min_av:254 cur_av:1  cm:1022 cc:1025
Mismatch     Block::0 bo:122 Volume offset:122  Expected:3 Got:2
new cur is 1024 from cur
Mismatch     Block::0 bo:61 Volume offset:61  Expected:1 Got:2
SPEC  v:0  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:1  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1025 from cur
SPEC  v:2  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1022 from min
Mismatch     Block::0 bo:62 Volume offset:62  Expected:1 Got:2
SPEC  v:255  min_av:254 cur_av:1  cm:1022 cc:1025
Mismatch     Block::0 bo:63 Volume offset:63  Expected:1 Got:2
new cur is 1023 from min
Mismatch     Block::0 bo:64 Volume offset:64  Expected:1 Got:2
SPEC  v:0  min_av:254 cur_av:1  cm:1022 cc:1025
Mismatch     Block::0 bo:65 Volume offset:65  Expected:1 Got:2
new cur is 1024 from cur
Mismatch     Block::0 bo:66 Volume offset:66  Expected:1 Got:2
SPEC  v:1  min_av:254 cur_av:1  cm:1022 cc:1025
Mismatch     Block::0 bo:67 Volume offset:67  Expected:1 Got:2
new cur is 1025 from cur
Mismatch     Block::0 bo:68 Volume offset:68  Expected:1 Got:2
SPEC  v:2  min_av:254 cur_av:1  cm:1022 cc:1025
Mismatch     Block::0 bo:69 Volume offset:69  Expected:1 Got:2
Mismatch     Block::0 bo:70 Volume offset:70  Expected:1 Got:2
Mismatch     Block::0 bo:123 Volume offset:123  Expected:3 Got:2
Mismatch     Block::0 bo:71 Volume offset:71  Expected:1 Got:2
Mismatch     Block::0 bo:72 Volume offset:72  Expected:1 Got:2
Mismatch     Block::0 bo:73 Volume offset:73  Expected:1 Got:2
Mismatch     Block::0 bo:124 Volume offset:124  Expected:3 Got:2
Mismatch     Block::0 bo:74 Volume offset:74  Expected:1 Got:2
Mismatch     Block::0 bo:125 Volume offset:125  Expected:3 Got:2
Mismatch     Block::0 bo:75 Volume offset:75  Expected:1 Got:2
Mismatch     Block::0 bo:126 Volume offset:126  Expected:3 Got:2
Mismatch     Block::0 bo:76 Volume offset:76  Expected:1 Got:2
Mismatch     Block::0 bo:127 Volume offset:127  Expected:3 Got:2
Mismatch     Block::0 bo:77 Volume offset:77  Expected:1 Got:2
Mismatch     Block::0 bo:128 Volume offset:128  Expected:3 Got:2
Mismatch     Block::0 bo:78 Volume offset:78  Expected:1 Got:2
Mismatch     Block::0 bo:129 Volume offset:129  Expected:3 Got:2
Mismatch     Block::0 bo:79 Volume offset:79  Expected:1 Got:2
Mismatch     Block::0 bo:130 Volume offset:130  Expected:3 Got:2
Mismatch     Block::0 bo:80 Volume offset:80  Expected:1 Got:2
Mismatch     Block::0 bo:131 Volume offset:131  Expected:3 Got:2
Mismatch     Block::0 bo:81 Volume offset:81  Expected:1 Got:2
Mismatch     Block::0 bo:132 Volume offset:132  Expected:3 Got:2
Mismatch     Block::0 bo:82 Volume offset:82  Expected:1 Got:2
Mismatch     Block::0 bo:133 Volume offset:133  Expected:3 Got:2
Mismatch     Block::0 bo:83 Volume offset:83  Expected:1 Got:2
Mismatch     Block::0 bo:134 Volume offset:134  Expected:3 Got:2
Mismatch     Block::0 bo:84 Volume offset:84  Expected:1 Got:2
Mismatch     Block::0 bo:135 Volume offset:135  Expected:3 Got:2
Mismatch     Block::0 bo:85 Volume offset:85  Expected:1 Got:2
Mismatch     Block::0 bo:136 Volume offset:136  Expected:3 Got:2
Mismatch     Block::0 bo:86 Volume offset:86  Expected:1 Got:2
Mismatch     Block::0 bo:137 Volume offset:137  Expected:3 Got:2
Mismatch     Block::0 bo:87 Volume offset:87  Expected:1 Got:2
Mismatch     Block::0 bo:138 Volume offset:138  Expected:3 Got:2
Mismatch     Block::0 bo:88 Volume offset:88  Expected:1 Got:2
Mismatch     Block::0 bo:139 Volume offset:139  Expected:3 Got:2
Mismatch     Block::0 bo:89 Volume offset:89  Expected:1 Got:2
Mismatch     Block::0 bo:140 Volume offset:140  Expected:3 Got:2
Mismatch     Block::0 bo:90 Volume offset:90  Expected:1 Got:2
Mismatch     Block::0 bo:141 Volume offset:141  Expected:3 Got:2
Mismatch     Block::0 bo:91 Volume offset:91  Expected:1 Got:2
Mismatch     Block::0 bo:142 Volume offset:142  Expected:3 Got:2
Mismatch     Block::0 bo:92 Volume offset:92  Expected:1 Got:2
Mismatch     Block::0 bo:143 Volume offset:143  Expected:3 Got:2
Mismatch     Block::0 bo:93 Volume offset:93  Expected:1 Got:2
Mismatch     Block::0 bo:144 Volume offset:144  Expected:3 Got:2
Mismatch     Block::0 bo:94 Volume offset:94  Expected:1 Got:2
Mismatch     Block::0 bo:145 Volume offset:145  Expected:3 Got:2
Mismatch     Block::0 bo:95 Volume offset:95  Expected:1 Got:2
Mismatch     Block::0 bo:146 Volume offset:146  Expected:3 Got:2
Mismatch     Block::0 bo:96 Volume offset:96  Expected:1 Got:2
Mismatch     Block::0 bo:147 Volume offset:147  Expected:3 Got:2
Mismatch     Block::0 bo:97 Volume offset:97  Expected:1 Got:2
Mismatch     Block::0 bo:148 Volume offset:148  Expected:3 Got:2
Mismatch     Block::0 bo:98 Volume offset:98  Expected:1 Got:2
Mismatch     Block::0 bo:149 Volume offset:149  Expected:3 Got:2
Mismatch     Block::0 bo:99 Volume offset:99  Expected:1 Got:2
Mismatch     Block::0 bo:150 Volume offset:150  Expected:3 Got:2
Mismatch     Block::0 bo:100 Volume offset:100  Expected:1 Got:2
Mismatch     Block::0 bo:151 Volume offset:151  Expected:3 Got:2
Mismatch     Block::0 bo:101 Volume offset:101  Expected:1 Got:2
Mismatch     Block::0 bo:152 Volume offset:152  Expected:3 Got:2
Mismatch     Block::0 bo:102 Volume offset:102  Expected:1 Got:2
Mismatch     Block::0 bo:153 Volume offset:153  Expected:3 Got:2
Mismatch     Block::0 bo:103 Volume offset:103  Expected:1 Got:2
Mismatch     Block::0 bo:154 Volume offset:154  Expected:3 Got:2
Mismatch     Block::0 bo:104 Volume offset:104  Expected:1 Got:2
Mismatch     Block::0 bo:155 Volume offset:155  Expected:3 Got:2
Mismatch     Block::0 bo:105 Volume offset:105  Expected:1 Got:2
Mismatch     Block::0 bo:156 Volume offset:156  Expected:3 Got:2
Mismatch     Block::0 bo:106 Volume offset:106  Expected:1 Got:2
Mismatch     Block::0 bo:157 Volume offset:157  Expected:3 Got:2
Mismatch     Block::0 bo:107 Volume offset:107  Expected:1 Got:2
Mismatch     Block::0 bo:158 Volume offset:158  Expected:3 Got:2
Mismatch     Block::0 bo:108 Volume offset:108  Expected:1 Got:2
Mismatch     Block::0 bo:159 Volume offset:159  Expected:3 Got:2
Mismatch     Block::0 bo:109 Volume offset:109  Expected:1 Got:2
Mismatch     Block::0 bo:160 Volume offset:160  Expected:3 Got:2
Mismatch     Block::0 bo:110 Volume offset:110  Expected:1 Got:2
Mismatch     Block::0 bo:161 Volume offset:161  Expected:3 Got:2
Mismatch     Block::0 bo:111 Volume offset:111  Expected:1 Got:2
Mismatch     Block::0 bo:162 Volume offset:162  Expected:3 Got:2
Mismatch     Block::0 bo:112 Volume offset:112  Expected:1 Got:2
Mismatch     Block::0 bo:163 Volume offset:163  Expected:3 Got:2
Mismatch     Block::0 bo:113 Volume offset:113  Expected:1 Got:2
Mismatch     Block::0 bo:164 Volume offset:164  Expected:3 Got:2
Mismatch     Block::0 bo:114 Volume offset:114  Expected:1 Got:2
Mismatch     Block::0 bo:165 Volume offset:165  Expected:3 Got:2
Mismatch     Block::0 bo:115 Volume offset:115  Expected:1 Got:2
Mismatch     Block::0 bo:166 Volume offset:166  Expected:3 Got:2
Mismatch     Block::0 bo:116 Volume offset:116  Expected:1 Got:2
Mismatch     Block::0 bo:167 Volume offset:167  Expected:3 Got:2
Mismatch     Block::0 bo:117 Volume offset:117  Expected:1 Got:2
Mismatch     Block::0 bo:168 Volume offset:168  Expected:3 Got:2
Mismatch     Block::0 bo:118 Volume offset:118  Expected:1 Got:2
Mismatch     Block::0 bo:169 Volume offset:169  Expected:3 Got:2
Mismatch     Block::0 bo:119 Volume offset:119  Expected:1 Got:2
Mismatch     Block::0 bo:170 Volume offset:170  Expected:3 Got:2
Mismatch     Block::0 bo:120 Volume offset:120  Expected:1 Got:2
Mismatch     Block::0 bo:171 Volume offset:171  Expected:3 Got:2
Mismatch     Block::0 bo:121 Volume offset:121  Expected:1 Got:2
Mismatch     Block::0 bo:172 Volume offset:172  Expected:3 Got:2
Mismatch     Block::0 bo:122 Volume offset:122  Expected:1 Got:2
Mismatch     Block::0 bo:173 Volume offset:173  Expected:3 Got:2
Mismatch     Block::0 bo:123 Volume offset:123  Expected:1 Got:2
Mismatch     Block::0 bo:174 Volume offset:174  Expected:3 Got:2
Mismatch     Block::0 bo:124 Volume offset:124  Expected:1 Got:2
Mismatch     Block::0 bo:175 Volume offset:175  Expected:3 Got:2
Mismatch     Block::0 bo:125 Volume offset:125  Expected:1 Got:2
Mismatch     Block::0 bo:176 Volume offset:176  Expected:3 Got:2
Mismatch     Block::0 bo:126 Volume offset:126  Expected:1 Got:2
Mismatch     Block::0 bo:177 Volume offset:177  Expected:3 Got:2
Mismatch     Block::0 bo:127 Volume offset:127  Expected:1 Got:2
Mismatch     Block::0 bo:178 Volume offset:178  Expected:3 Got:2
Mismatch     Block::0 bo:128 Volume offset:128  Expected:1 Got:2
Mismatch     Block::0 bo:179 Volume offset:179  Expected:3 Got:2
Mismatch     Block::0 bo:129 Volume offset:129  Expected:1 Got:2
Mismatch     Block::0 bo:180 Volume offset:180  Expected:3 Got:2
Mismatch     Block::0 bo:130 Volume offset:130  Expected:1 Got:2
Mismatch     Block::0 bo:181 Volume offset:181  Expected:3 Got:2
Mismatch     Block::0 bo:131 Volume offset:131  Expected:1 Got:2
Mismatch     Block::0 bo:182 Volume offset:182  Expected:3 Got:2
Mismatch     Block::0 bo:132 Volume offset:132  Expected:1 Got:2
Mismatch     Block::0 bo:183 Volume offset:183  Expected:3 Got:2
Mismatch     Block::0 bo:133 Volume offset:133  Expected:1 Got:2
Mismatch     Block::0 bo:184 Volume offset:184  Expected:3 Got:2
Mismatch     Block::0 bo:134 Volume offset:134  Expected:1 Got:2
Mismatch     Block::0 bo:185 Volume offset:185  Expected:3 Got:2
Mismatch     Block::0 bo:135 Volume offset:135  Expected:1 Got:2
Mismatch     Block::0 bo:186 Volume offset:186  Expected:3 Got:2
Mismatch     Block::0 bo:136 Volume offset:136  Expected:1 Got:2
Mismatch     Block::0 bo:187 Volume offset:187  Expected:3 Got:2
Mismatch     Block::0 bo:137 Volume offset:137  Expected:1 Got:2
Mismatch     Block::0 bo:188 Volume offset:188  Expected:3 Got:2
Mismatch     Block::0 bo:138 Volume offset:138  Expected:1 Got:2
Mismatch     Block::0 bo:189 Volume offset:189  Expected:3 Got:2
Mismatch     Block::0 bo:139 Volume offset:139  Expected:1 Got:2
Mismatch     Block::0 bo:190 Volume offset:190  Expected:3 Got:2
Mismatch     Block::0 bo:140 Volume offset:140  Expected:1 Got:2
Mismatch     Block::0 bo:191 Volume offset:191  Expected:3 Got:2
Mismatch     Block::0 bo:141 Volume offset:141  Expected:1 Got:2
Mismatch     Block::0 bo:192 Volume offset:192  Expected:3 Got:2
Mismatch     Block::0 bo:142 Volume offset:142  Expected:1 Got:2
Mismatch     Block::0 bo:193 Volume offset:193  Expected:3 Got:2
Mismatch     Block::0 bo:143 Volume offset:143  Expected:1 Got:2
Mismatch     Block::0 bo:194 Volume offset:194  Expected:3 Got:2
Mismatch     Block::0 bo:144 Volume offset:144  Expected:1 Got:2
Mismatch     Block::0 bo:195 Volume offset:195  Expected:3 Got:2
Mismatch     Block::0 bo:145 Volume offset:145  Expected:1 Got:2
Mismatch     Block::0 bo:196 Volume offset:196  Expected:3 Got:2
Mismatch     Block::0 bo:146 Volume offset:146  Expected:1 Got:2
Mismatch     Block::0 bo:197 Volume offset:197  Expected:3 Got:2
Mismatch     Block::0 bo:147 Volume offset:147  Expected:1 Got:2
Mismatch     Block::0 bo:198 Volume offset:198  Expected:3 Got:2
Mismatch     Block::0 bo:148 Volume offset:148  Expected:1 Got:2
Mismatch     Block::0 bo:199 Volume offset:199  Expected:3 Got:2
Mismatch     Block::0 bo:149 Volume offset:149  Expected:1 Got:2
Mismatch     Block::0 bo:200 Volume offset:200  Expected:3 Got:2
Mismatch     Block::0 bo:150 Volume offset:150  Expected:1 Got:2
Mismatch     Block::0 bo:201 Volume offset:201  Expected:3 Got:2
Mismatch     Block::0 bo:151 Volume offset:151  Expected:1 Got:2
Mismatch     Block::0 bo:202 Volume offset:202  Expected:3 Got:2
Mismatch     Block::0 bo:152 Volume offset:152  Expected:1 Got:2
Mismatch     Block::0 bo:203 Volume offset:203  Expected:3 Got:2
Mismatch     Block::0 bo:153 Volume offset:153  Expected:1 Got:2
Mismatch     Block::0 bo:204 Volume offset:204  Expected:3 Got:2
Mismatch     Block::0 bo:154 Volume offset:154  Expected:1 Got:2
Mismatch     Block::0 bo:205 Volume offset:205  Expected:3 Got:2
Mismatch     Block::0 bo:155 Volume offset:155  Expected:1 Got:2
Mismatch     Block::0 bo:206 Volume offset:206  Expected:3 Got:2
Mismatch     Block::0 bo:156 Volume offset:156  Expected:1 Got:2
Mismatch     Block::0 bo:207 Volume offset:207  Expected:3 Got:2
Mismatch     Block::0 bo:157 Volume offset:157  Expected:1 Got:2
Mismatch     Block::0 bo:208 Volume offset:208  Expected:3 Got:2
Mismatch     Block::0 bo:158 Volume offset:158  Expected:1 Got:2
Mismatch     Block::0 bo:209 Volume offset:209  Expected:3 Got:2
Mismatch     Block::0 bo:159 Volume offset:159  Expected:1 Got:2
Mismatch     Block::0 bo:210 Volume offset:210  Expected:3 Got:2
Mismatch     Block::0 bo:160 Volume offset:160  Expected:1 Got:2
Mismatch     Block::0 bo:211 Volume offset:211  Expected:3 Got:2
Mismatch     Block::0 bo:161 Volume offset:161  Expected:1 Got:2
Mismatch     Block::0 bo:212 Volume offset:212  Expected:3 Got:2
Mismatch     Block::0 bo:162 Volume offset:162  Expected:1 Got:2
Mismatch     Block::0 bo:213 Volume offset:213  Expected:3 Got:2
Mismatch     Block::0 bo:163 Volume offset:163  Expected:1 Got:2
Mismatch     Block::0 bo:214 Volume offset:214  Expected:3 Got:2
Mismatch     Block::0 bo:164 Volume offset:164  Expected:1 Got:2
Mismatch     Block::0 bo:215 Volume offset:215  Expected:3 Got:2
Mismatch     Block::0 bo:165 Volume offset:165  Expected:1 Got:2
Mismatch     Block::0 bo:216 Volume offset:216  Expected:3 Got:2
Mismatch     Block::0 bo:166 Volume offset:166  Expected:1 Got:2
Mismatch     Block::0 bo:217 Volume offset:217  Expected:3 Got:2
Mismatch     Block::0 bo:167 Volume offset:167  Expected:1 Got:2
Mismatch     Block::0 bo:218 Volume offset:218  Expected:3 Got:2
Mismatch     Block::0 bo:168 Volume offset:168  Expected:1 Got:2
Mismatch     Block::0 bo:219 Volume offset:219  Expected:3 Got:2
Mismatch     Block::0 bo:169 Volume offset:169  Expected:1 Got:2
Mismatch     Block::0 bo:220 Volume offset:220  Expected:3 Got:2
Mismatch     Block::0 bo:170 Volume offset:170  Expected:1 Got:2
Mismatch     Block::0 bo:221 Volume offset:221  Expected:3 Got:2
Mismatch     Block::0 bo:171 Volume offset:171  Expected:1 Got:2
Mismatch     Block::0 bo:222 Volume offset:222  Expected:3 Got:2
Mismatch     Block::0 bo:172 Volume offset:172  Expected:1 Got:2
Mismatch     Block::0 bo:223 Volume offset:223  Expected:3 Got:2
Mismatch     Block::0 bo:173 Volume offset:173  Expected:1 Got:2
Mismatch     Block::0 bo:224 Volume offset:224  Expected:3 Got:2
Mismatch     Block::0 bo:174 Volume offset:174  Expected:1 Got:2
Mismatch     Block::0 bo:225 Volume offset:225  Expected:3 Got:2
Mismatch     Block::0 bo:175 Volume offset:175  Expected:1 Got:2
Mismatch     Block::0 bo:226 Volume offset:226  Expected:3 Got:2
Mismatch     Block::0 bo:176 Volume offset:176  Expected:1 Got:2
Mismatch     Block::0 bo:227 Volume offset:227  Expected:3 Got:2
Mismatch     Block::0 bo:177 Volume offset:177  Expected:1 Got:2
Mismatch     Block::0 bo:228 Volume offset:228  Expected:3 Got:2
Mismatch     Block::0 bo:178 Volume offset:178  Expected:1 Got:2
Mismatch     Block::0 bo:229 Volume offset:229  Expected:3 Got:2
Mismatch     Block::0 bo:179 Volume offset:179  Expected:1 Got:2
Mismatch     Block::0 bo:230 Volume offset:230  Expected:3 Got:2
Mismatch     Block::0 bo:180 Volume offset:180  Expected:1 Got:2
Mismatch     Block::0 bo:231 Volume offset:231  Expected:3 Got:2
Mismatch     Block::0 bo:181 Volume offset:181  Expected:1 Got:2
Mismatch     Block::0 bo:232 Volume offset:232  Expected:3 Got:2
Mismatch     Block::0 bo:182 Volume offset:182  Expected:1 Got:2
Mismatch     Block::0 bo:233 Volume offset:233  Expected:3 Got:2
Mismatch     Block::0 bo:183 Volume offset:183  Expected:1 Got:2
Mismatch     Block::0 bo:234 Volume offset:234  Expected:3 Got:2
Mismatch     Block::0 bo:184 Volume offset:184  Expected:1 Got:2
Mismatch     Block::0 bo:235 Volume offset:235  Expected:3 Got:2
Mismatch     Block::0 bo:185 Volume offset:185  Expected:1 Got:2
Mismatch     Block::0 bo:236 Volume offset:236  Expected:3 Got:2
Mismatch     Block::0 bo:186 Volume offset:186  Expected:1 Got:2
Mismatch     Block::0 bo:237 Volume offset:237  Expected:3 Got:2
Mismatch     Block::0 bo:187 Volume offset:187  Expected:1 Got:2
Mismatch     Block::0 bo:238 Volume offset:238  Expected:3 Got:2
Mismatch     Block::0 bo:188 Volume offset:188  Expected:1 Got:2
Mismatch     Block::0 bo:239 Volume offset:239  Expected:3 Got:2
Mismatch     Block::0 bo:189 Volume offset:189  Expected:1 Got:2
Mismatch     Block::0 bo:240 Volume offset:240  Expected:3 Got:2
Mismatch     Block::0 bo:190 Volume offset:190  Expected:1 Got:2
Mismatch     Block::0 bo:241 Volume offset:241  Expected:3 Got:2
Mismatch     Block::0 bo:191 Volume offset:191  Expected:1 Got:2
Mismatch     Block::0 bo:242 Volume offset:242  Expected:3 Got:2
Mismatch     Block::0 bo:192 Volume offset:192  Expected:1 Got:2
Mismatch     Block::0 bo:243 Volume offset:243  Expected:3 Got:2
Mismatch     Block::0 bo:193 Volume offset:193  Expected:1 Got:2
Mismatch     Block::0 bo:244 Volume offset:244  Expected:3 Got:2
Mismatch     Block::0 bo:194 Volume offset:194  Expected:1 Got:2
Mismatch     Block::0 bo:245 Volume offset:245  Expected:3 Got:2
Mismatch     Block::0 bo:195 Volume offset:195  Expected:1 Got:2
Mismatch     Block::0 bo:246 Volume offset:246  Expected:3 Got:2
Mismatch     Block::0 bo:196 Volume offset:196  Expected:1 Got:2
Mismatch     Block::0 bo:247 Volume offset:247  Expected:3 Got:2
Mismatch     Block::0 bo:197 Volume offset:197  Expected:1 Got:2
Mismatch     Block::0 bo:248 Volume offset:248  Expected:3 Got:2
Mismatch     Block::0 bo:198 Volume offset:198  Expected:1 Got:2
Mismatch     Block::0 bo:249 Volume offset:249  Expected:3 Got:2
Mismatch     Block::0 bo:199 Volume offset:199  Expected:1 Got:2
Mismatch     Block::0 bo:250 Volume offset:250  Expected:3 Got:2
Mismatch     Block::0 bo:200 Volume offset:200  Expected:1 Got:2
Mismatch     Block::0 bo:251 Volume offset:251  Expected:3 Got:2
Mismatch     Block::0 bo:201 Volume offset:201  Expected:1 Got:2
Mismatch     Block::0 bo:252 Volume offset:252  Expected:3 Got:2
Mismatch     Block::0 bo:202 Volume offset:202  Expected:1 Got:2
Mismatch     Block::0 bo:253 Volume offset:253  Expected:3 Got:2
Mismatch     Block::0 bo:203 Volume offset:203  Expected:1 Got:2
Mismatch     Block::0 bo:254 Volume offset:254  Expected:3 Got:2
Mismatch     Block::0 bo:204 Volume offset:204  Expected:1 Got:2
Mismatch     Block::0 bo:255 Volume offset:255  Expected:3 Got:2
Mismatch     Block::0 bo:205 Volume offset:205  Expected:1 Got:2
Mismatch     Block::0 bo:256 Volume offset:256  Expected:3 Got:2
Mismatch     Block::0 bo:206 Volume offset:206  Expected:1 Got:2
Mismatch     Block::0 bo:257 Volume offset:257  Expected:3 Got:2
Mismatch     Block::0 bo:207 Volume offset:207  Expected:1 Got:2
Mismatch     Block::0 bo:258 Volume offset:258  Expected:3 Got:2
Mismatch     Block::0 bo:208 Volume offset:208  Expected:1 Got:2
Mismatch     Block::0 bo:259 Volume offset:259  Expected:3 Got:2
Mismatch     Block::0 bo:209 Volume offset:209  Expected:1 Got:2
Mismatch     Block::0 bo:260 Volume offset:260  Expected:3 Got:2
Mismatch     Block::0 bo:210 Volume offset:210  Expected:1 Got:2
Mismatch     Block::0 bo:261 Volume offset:261  Expected:3 Got:2
Mismatch     Block::0 bo:211 Volume offset:211  Expected:1 Got:2
Mismatch     Block::0 bo:262 Volume offset:262  Expected:3 Got:2
Mismatch     Block::0 bo:212 Volume offset:212  Expected:1 Got:2
Mismatch     Block::0 bo:263 Volume offset:263  Expected:3 Got:2
Mismatch     Block::0 bo:213 Volume offset:213  Expected:1 Got:2
Mismatch     Block::0 bo:264 Volume offset:264  Expected:3 Got:2
Mismatch     Block::0 bo:214 Volume offset:214  Expected:1 Got:2
Mismatch     Block::0 bo:265 Volume offset:265  Expected:3 Got:2
Mismatch     Block::0 bo:215 Volume offset:215  Expected:1 Got:2
Mismatch     Block::0 bo:266 Volume offset:266  Expected:3 Got:2
Mismatch     Block::0 bo:216 Volume offset:216  Expected:1 Got:2
Mismatch     Block::0 bo:267 Volume offset:267  Expected:3 Got:2
Mismatch     Block::0 bo:217 Volume offset:217  Expected:1 Got:2
Mismatch     Block::0 bo:268 Volume offset:268  Expected:3 Got:2
Mismatch     Block::0 bo:218 Volume offset:218  Expected:1 Got:2
Mismatch     Block::0 bo:269 Volume offset:269  Expected:3 Got:2
Mismatch     Block::0 bo:219 Volume offset:219  Expected:1 Got:2
Mismatch     Block::0 bo:270 Volume offset:270  Expected:3 Got:2
Mismatch     Block::0 bo:220 Volume offset:220  Expected:1 Got:2
Mismatch     Block::0 bo:271 Volume offset:271  Expected:3 Got:2
Mismatch     Block::0 bo:221 Volume offset:221  Expected:1 Got:2
Mismatch     Block::0 bo:272 Volume offset:272  Expected:3 Got:2
Mismatch     Block::0 bo:222 Volume offset:222  Expected:1 Got:2
Mismatch     Block::0 bo:273 Volume offset:273  Expected:3 Got:2
Mismatch     Block::0 bo:223 Volume offset:223  Expected:1 Got:2
Mismatch     Block::0 bo:274 Volume offset:274  Expected:3 Got:2
Mismatch     Block::0 bo:224 Volume offset:224  Expected:1 Got:2
Mismatch     Block::0 bo:275 Volume offset:275  Expected:3 Got:2
Mismatch     Block::0 bo:225 Volume offset:225  Expected:1 Got:2
Mismatch     Block::0 bo:276 Volume offset:276  Expected:3 Got:2
Mismatch     Block::0 bo:226 Volume offset:226  Expected:1 Got:2
Mismatch     Block::0 bo:277 Volume offset:277  Expected:3 Got:2
Mismatch     Block::0 bo:227 Volume offset:227  Expected:1 Got:2
Mismatch     Block::0 bo:278 Volume offset:278  Expected:3 Got:2
Mismatch     Block::0 bo:228 Volume offset:228  Expected:1 Got:2
Mismatch     Block::0 bo:279 Volume offset:279  Expected:3 Got:2
Mismatch     Block::0 bo:229 Volume offset:229  Expected:1 Got:2
Mismatch     Block::0 bo:280 Volume offset:280  Expected:3 Got:2
Mismatch     Block::0 bo:230 Volume offset:230  Expected:1 Got:2
Mismatch     Block::0 bo:281 Volume offset:281  Expected:3 Got:2
Mismatch     Block::0 bo:231 Volume offset:231  Expected:1 Got:2
Mismatch     Block::0 bo:282 Volume offset:282  Expected:3 Got:2
Mismatch     Block::0 bo:232 Volume offset:232  Expected:1 Got:2
Mismatch     Block::0 bo:283 Volume offset:283  Expected:3 Got:2
Mismatch     Block::0 bo:233 Volume offset:233  Expected:1 Got:2
Mismatch     Block::0 bo:284 Volume offset:284  Expected:3 Got:2
Mismatch     Block::0 bo:234 Volume offset:234  Expected:1 Got:2
Mismatch     Block::0 bo:285 Volume offset:285  Expected:3 Got:2
Mismatch     Block::0 bo:235 Volume offset:235  Expected:1 Got:2
Mismatch     Block::0 bo:286 Volume offset:286  Expected:3 Got:2
Mismatch     Block::0 bo:236 Volume offset:236  Expected:1 Got:2
Mismatch     Block::0 bo:287 Volume offset:287  Expected:3 Got:2
Mismatch     Block::0 bo:237 Volume offset:237  Expected:1 Got:2
Mismatch     Block::0 bo:288 Volume offset:288  Expected:3 Got:2
Mismatch     Block::0 bo:238 Volume offset:238  Expected:1 Got:2
Mismatch     Block::0 bo:289 Volume offset:289  Expected:3 Got:2
Mismatch     Block::0 bo:239 Volume offset:239  Expected:1 Got:2
Mismatch     Block::0 bo:290 Volume offset:290  Expected:3 Got:2
Mismatch     Block::0 bo:240 Volume offset:240  Expected:1 Got:2
Mismatch     Block::0 bo:291 Volume offset:291  Expected:3 Got:2
Mismatch     Block::0 bo:241 Volume offset:241  Expected:1 Got:2
Mismatch     Block::0 bo:292 Volume offset:292  Expected:3 Got:2
Mismatch     Block::0 bo:242 Volume offset:242  Expected:1 Got:2
Mismatch     Block::0 bo:293 Volume offset:293  Expected:3 Got:2
Mismatch     Block::0 bo:243 Volume offset:243  Expected:1 Got:2
Mismatch     Block::0 bo:294 Volume offset:294  Expected:3 Got:2
Mismatch     Block::0 bo:244 Volume offset:244  Expected:1 Got:2
Mismatch     Block::0 bo:295 Volume offset:295  Expected:3 Got:2
Mismatch     Block::0 bo:245 Volume offset:245  Expected:1 Got:2
Mismatch     Block::0 bo:296 Volume offset:296  Expected:3 Got:2
Mismatch     Block::0 bo:246 Volume offset:246  Expected:1 Got:2
Mismatch     Block::0 bo:297 Volume offset:297  Expected:3 Got:2
Mismatch     Block::0 bo:247 Volume offset:247  Expected:1 Got:2
Mismatch     Block::0 bo:298 Volume offset:298  Expected:3 Got:2
Mismatch     Block::0 bo:248 Volume offset:248  Expected:1 Got:2
Mismatch     Block::0 bo:299 Volume offset:299  Expected:3 Got:2
Mismatch     Block::0 bo:249 Volume offset:249  Expected:1 Got:2
Mismatch     Block::0 bo:300 Volume offset:300  Expected:3 Got:2
Mismatch     Block::0 bo:250 Volume offset:250  Expected:1 Got:2
Mismatch     Block::0 bo:301 Volume offset:301  Expected:3 Got:2
Mismatch     Block::0 bo:251 Volume offset:251  Expected:1 Got:2
Mismatch     Block::0 bo:302 Volume offset:302  Expected:3 Got:2
Mismatch     Block::0 bo:252 Volume offset:252  Expected:1 Got:2
Mismatch     Block::0 bo:303 Volume offset:303  Expected:3 Got:2
Mismatch     Block::0 bo:253 Volume offset:253  Expected:1 Got:2
Mismatch     Block::0 bo:304 Volume offset:304  Expected:3 Got:2
Mismatch     Block::0 bo:254 Volume offset:254  Expected:1 Got:2
Mismatch     Block::0 bo:305 Volume offset:305  Expected:3 Got:2
Mismatch     Block::0 bo:255 Volume offset:255  Expected:1 Got:2
Mismatch     Block::0 bo:306 Volume offset:306  Expected:3 Got:2
Mismatch     Block::0 bo:256 Volume offset:256  Expected:1 Got:2
Mismatch     Block::0 bo:307 Volume offset:307  Expected:3 Got:2
Mismatch     Block::0 bo:257 Volume offset:257  Expected:1 Got:2
Mismatch     Block::0 bo:308 Volume offset:308  Expected:3 Got:2
Mismatch     Block::0 bo:258 Volume offset:258  Expected:1 Got:2
Mismatch     Block::0 bo:309 Volume offset:309  Expected:3 Got:2
Mismatch     Block::0 bo:259 Volume offset:259  Expected:1 Got:2
Mismatch     Block::0 bo:310 Volume offset:310  Expected:3 Got:2
Mismatch     Block::0 bo:260 Volume offset:260  Expected:1 Got:2
Mismatch     Block::0 bo:311 Volume offset:311  Expected:3 Got:2
Mismatch     Block::0 bo:261 Volume offset:261  Expected:1 Got:2
Mismatch     Block::0 bo:312 Volume offset:312  Expected:3 Got:2
Mismatch     Block::0 bo:262 Volume offset:262  Expected:1 Got:2
Mismatch     Block::0 bo:313 Volume offset:313  Expected:3 Got:2
Mismatch     Block::0 bo:263 Volume offset:263  Expected:1 Got:2
Mismatch     Block::0 bo:314 Volume offset:314  Expected:3 Got:2
Mismatch     Block::0 bo:264 Volume offset:264  Expected:1 Got:2
Mismatch     Block::0 bo:315 Volume offset:315  Expected:3 Got:2
Mismatch     Block::0 bo:265 Volume offset:265  Expected:1 Got:2
Mismatch     Block::0 bo:316 Volume offset:316  Expected:3 Got:2
Mismatch     Block::0 bo:266 Volume offset:266  Expected:1 Got:2
Mismatch     Block::0 bo:317 Volume offset:317  Expected:3 Got:2
Mismatch     Block::0 bo:267 Volume offset:267  Expected:1 Got:2
Mismatch     Block::0 bo:318 Volume offset:318  Expected:3 Got:2
Mismatch     Block::0 bo:268 Volume offset:268  Expected:1 Got:2
Mismatch     Block::0 bo:319 Volume offset:319  Expected:3 Got:2
Mismatch     Block::0 bo:269 Volume offset:269  Expected:1 Got:2
Mismatch     Block::0 bo:320 Volume offset:320  Expected:3 Got:2
Mismatch     Block::0 bo:270 Volume offset:270  Expected:1 Got:2
Mismatch     Block::0 bo:321 Volume offset:321  Expected:3 Got:2
Mismatch     Block::0 bo:271 Volume offset:271  Expected:1 Got:2
Mismatch     Block::0 bo:322 Volume offset:322  Expected:3 Got:2
Mismatch     Block::0 bo:272 Volume offset:272  Expected:1 Got:2
Mismatch     Block::0 bo:323 Volume offset:323  Expected:3 Got:2
Mismatch     Block::0 bo:273 Volume offset:273  Expected:1 Got:2
Mismatch     Block::0 bo:324 Volume offset:324  Expected:3 Got:2
Mismatch     Block::0 bo:274 Volume offset:274  Expected:1 Got:2
Mismatch     Block::0 bo:325 Volume offset:325  Expected:3 Got:2
Mismatch     Block::0 bo:275 Volume offset:275  Expected:1 Got:2
Mismatch     Block::0 bo:326 Volume offset:326  Expected:3 Got:2
Mismatch     Block::0 bo:276 Volume offset:276  Expected:1 Got:2
Mismatch     Block::0 bo:327 Volume offset:327  Expected:3 Got:2
Mismatch     Block::0 bo:277 Volume offset:277  Expected:1 Got:2
Mismatch     Block::0 bo:328 Volume offset:328  Expected:3 Got:2
Mismatch     Block::0 bo:278 Volume offset:278  Expected:1 Got:2
Mismatch     Block::0 bo:329 Volume offset:329  Expected:3 Got:2
Mismatch     Block::0 bo:279 Volume offset:279  Expected:1 Got:2
Mismatch     Block::0 bo:330 Volume offset:330  Expected:3 Got:2
Mismatch     Block::0 bo:280 Volume offset:280  Expected:1 Got:2
Mismatch     Block::0 bo:331 Volume offset:331  Expected:3 Got:2
Mismatch     Block::0 bo:281 Volume offset:281  Expected:1 Got:2
Mismatch     Block::0 bo:332 Volume offset:332  Expected:3 Got:2
Mismatch     Block::0 bo:282 Volume offset:282  Expected:1 Got:2
Mismatch     Block::0 bo:333 Volume offset:333  Expected:3 Got:2
Mismatch     Block::0 bo:283 Volume offset:283  Expected:1 Got:2
Mismatch     Block::0 bo:334 Volume offset:334  Expected:3 Got:2
Mismatch     Block::0 bo:284 Volume offset:284  Expected:1 Got:2
Mismatch     Block::0 bo:335 Volume offset:335  Expected:3 Got:2
Mismatch     Block::0 bo:285 Volume offset:285  Expected:1 Got:2
Mismatch     Block::0 bo:336 Volume offset:336  Expected:3 Got:2
Mismatch     Block::0 bo:286 Volume offset:286  Expected:1 Got:2
Mismatch     Block::0 bo:337 Volume offset:337  Expected:3 Got:2
Mismatch     Block::0 bo:287 Volume offset:287  Expected:1 Got:2
Mismatch     Block::0 bo:338 Volume offset:338  Expected:3 Got:2
Mismatch     Block::0 bo:288 Volume offset:288  Expected:1 Got:2
Mismatch     Block::0 bo:339 Volume offset:339  Expected:3 Got:2
Mismatch     Block::0 bo:289 Volume offset:289  Expected:1 Got:2
Mismatch     Block::0 bo:340 Volume offset:340  Expected:3 Got:2
Mismatch     Block::0 bo:290 Volume offset:290  Expected:1 Got:2
Mismatch     Block::0 bo:341 Volume offset:341  Expected:3 Got:2
Mismatch     Block::0 bo:291 Volume offset:291  Expected:1 Got:2
Mismatch     Block::0 bo:342 Volume offset:342  Expected:3 Got:2
Mismatch     Block::0 bo:292 Volume offset:292  Expected:1 Got:2
Mismatch     Block::0 bo:343 Volume offset:343  Expected:3 Got:2
Mismatch     Block::0 bo:293 Volume offset:293  Expected:1 Got:2
Mismatch     Block::0 bo:344 Volume offset:344  Expected:3 Got:2
Mismatch     Block::0 bo:294 Volume offset:294  Expected:1 Got:2
Mismatch     Block::0 bo:345 Volume offset:345  Expected:3 Got:2
Mismatch     Block::0 bo:295 Volume offset:295  Expected:1 Got:2
Mismatch     Block::0 bo:346 Volume offset:346  Expected:3 Got:2
Mismatch     Block::0 bo:296 Volume offset:296  Expected:1 Got:2
Mismatch     Block::0 bo:347 Volume offset:347  Expected:3 Got:2
Mismatch     Block::0 bo:297 Volume offset:297  Expected:1 Got:2
Mismatch     Block::0 bo:348 Volume offset:348  Expected:3 Got:2
Mismatch     Block::0 bo:298 Volume offset:298  Expected:1 Got:2
Mismatch     Block::0 bo:349 Volume offset:349  Expected:3 Got:2
Mismatch     Block::0 bo:299 Volume offset:299  Expected:1 Got:2
Mismatch     Block::0 bo:350 Volume offset:350  Expected:3 Got:2
Mismatch     Block::0 bo:300 Volume offset:300  Expected:1 Got:2
Mismatch     Block::0 bo:351 Volume offset:351  Expected:3 Got:2
Mismatch     Block::0 bo:301 Volume offset:301  Expected:1 Got:2
Mismatch     Block::0 bo:352 Volume offset:352  Expected:3 Got:2
Mismatch     Block::0 bo:302 Volume offset:302  Expected:1 Got:2
Mismatch     Block::0 bo:353 Volume offset:353  Expected:3 Got:2
Mismatch     Block::0 bo:303 Volume offset:303  Expected:1 Got:2
Mismatch     Block::0 bo:354 Volume offset:354  Expected:3 Got:2
Mismatch     Block::0 bo:304 Volume offset:304  Expected:1 Got:2
Mismatch     Block::0 bo:355 Volume offset:355  Expected:3 Got:2
Mismatch     Block::0 bo:305 Volume offset:305  Expected:1 Got:2
Mismatch     Block::0 bo:356 Volume offset:356  Expected:3 Got:2
Mismatch     Block::0 bo:306 Volume offset:306  Expected:1 Got:2
Mismatch     Block::0 bo:357 Volume offset:357  Expected:3 Got:2
Mismatch     Block::0 bo:307 Volume offset:307  Expected:1 Got:2
Mismatch     Block::0 bo:358 Volume offset:358  Expected:3 Got:2
Mismatch     Block::0 bo:308 Volume offset:308  Expected:1 Got:2
Mismatch     Block::0 bo:359 Volume offset:359  Expected:3 Got:2
Mismatch     Block::0 bo:309 Volume offset:309  Expected:1 Got:2
Mismatch     Block::0 bo:360 Volume offset:360  Expected:3 Got:2
Mismatch     Block::0 bo:310 Volume offset:310  Expected:1 Got:2
Mismatch     Block::0 bo:361 Volume offset:361  Expected:3 Got:2
Mismatch     Block::0 bo:311 Volume offset:311  Expected:1 Got:2
Mismatch     Block::0 bo:362 Volume offset:362  Expected:3 Got:2
Mismatch     Block::0 bo:312 Volume offset:312  Expected:1 Got:2
Mismatch     Block::0 bo:363 Volume offset:363  Expected:3 Got:2
Mismatch     Block::0 bo:313 Volume offset:313  Expected:1 Got:2
Mismatch     Block::0 bo:364 Volume offset:364  Expected:3 Got:2
Mismatch     Block::0 bo:314 Volume offset:314  Expected:1 Got:2
Mismatch     Block::0 bo:365 Volume offset:365  Expected:3 Got:2
Mismatch     Block::0 bo:315 Volume offset:315  Expected:1 Got:2
Mismatch     Block::0 bo:366 Volume offset:366  Expected:3 Got:2
Mismatch     Block::0 bo:316 Volume offset:316  Expected:1 Got:2
Mismatch     Block::0 bo:367 Volume offset:367  Expected:3 Got:2
Mismatch     Block::0 bo:317 Volume offset:317  Expected:1 Got:2
Mismatch     Block::0 bo:368 Volume offset:368  Expected:3 Got:2
Mismatch     Block::0 bo:318 Volume offset:318  Expected:1 Got:2
Mismatch     Block::0 bo:369 Volume offset:369  Expected:3 Got:2
Mismatch     Block::0 bo:319 Volume offset:319  Expected:1 Got:2
Mismatch     Block::0 bo:370 Volume offset:370  Expected:3 Got:2
Mismatch     Block::0 bo:320 Volume offset:320  Expected:1 Got:2
Mismatch     Block::0 bo:371 Volume offset:371  Expected:3 Got:2
Mismatch     Block::0 bo:321 Volume offset:321  Expected:1 Got:2
Mismatch     Block::0 bo:372 Volume offset:372  Expected:3 Got:2
Mismatch     Block::0 bo:322 Volume offset:322  Expected:1 Got:2
Mismatch     Block::0 bo:373 Volume offset:373  Expected:3 Got:2
Mismatch     Block::0 bo:323 Volume offset:323  Expected:1 Got:2
Mismatch     Block::0 bo:374 Volume offset:374  Expected:3 Got:2
Mismatch     Block::0 bo:324 Volume offset:324  Expected:1 Got:2
Mismatch     Block::0 bo:375 Volume offset:375  Expected:3 Got:2
Mismatch     Block::0 bo:325 Volume offset:325  Expected:1 Got:2
Mismatch     Block::0 bo:376 Volume offset:376  Expected:3 Got:2
Mismatch     Block::0 bo:326 Volume offset:326  Expected:1 Got:2
Mismatch     Block::0 bo:377 Volume offset:377  Expected:3 Got:2
Mismatch     Block::0 bo:327 Volume offset:327  Expected:1 Got:2
Mismatch     Block::0 bo:378 Volume offset:378  Expected:3 Got:2
Mismatch     Block::0 bo:328 Volume offset:328  Expected:1 Got:2
Mismatch     Block::0 bo:379 Volume offset:379  Expected:3 Got:2
Mismatch     Block::0 bo:329 Volume offset:329  Expected:1 Got:2
Mismatch     Block::0 bo:380 Volume offset:380  Expected:3 Got:2
Mismatch     Block::0 bo:330 Volume offset:330  Expected:1 Got:2
Mismatch     Block::0 bo:381 Volume offset:381  Expected:3 Got:2
Mismatch     Block::0 bo:331 Volume offset:331  Expected:1 Got:2
Mismatch     Block::0 bo:382 Volume offset:382  Expected:3 Got:2
Mismatch     Block::0 bo:332 Volume offset:332  Expected:1 Got:2
Mismatch     Block::0 bo:383 Volume offset:383  Expected:3 Got:2
Mismatch     Block::0 bo:333 Volume offset:333  Expected:1 Got:2
Mismatch     Block::0 bo:384 Volume offset:384  Expected:3 Got:2
Mismatch     Block::0 bo:334 Volume offset:334  Expected:1 Got:2
Mismatch     Block::0 bo:385 Volume offset:385  Expected:3 Got:2
Mismatch     Block::0 bo:335 Volume offset:335  Expected:1 Got:2
Mismatch     Block::0 bo:386 Volume offset:386  Expected:3 Got:2
Mismatch     Block::0 bo:336 Volume offset:336  Expected:1 Got:2
Mismatch     Block::0 bo:387 Volume offset:387  Expected:3 Got:2
Mismatch     Block::0 bo:337 Volume offset:337  Expected:1 Got:2
Mismatch     Block::0 bo:388 Volume offset:388  Expected:3 Got:2
Mismatch     Block::0 bo:338 Volume offset:338  Expected:1 Got:2
Mismatch     Block::0 bo:389 Volume offset:389  Expected:3 Got:2
Mismatch     Block::0 bo:339 Volume offset:339  Expected:1 Got:2
Mismatch     Block::0 bo:390 Volume offset:390  Expected:3 Got:2
Mismatch     Block::0 bo:340 Volume offset:340  Expected:1 Got:2
Mismatch     Block::0 bo:391 Volume offset:391  Expected:3 Got:2
Mismatch     Block::0 bo:341 Volume offset:341  Expected:1 Got:2
Mismatch     Block::0 bo:392 Volume offset:392  Expected:3 Got:2
Mismatch     Block::0 bo:342 Volume offset:342  Expected:1 Got:2
Mismatch     Block::0 bo:393 Volume offset:393  Expected:3 Got:2
Mismatch     Block::0 bo:343 Volume offset:343  Expected:1 Got:2
Mismatch     Block::0 bo:394 Volume offset:394  Expected:3 Got:2
Mismatch     Block::0 bo:344 Volume offset:344  Expected:1 Got:2
Mismatch     Block::0 bo:395 Volume offset:395  Expected:3 Got:2
Mismatch     Block::0 bo:345 Volume offset:345  Expected:1 Got:2
Mismatch     Block::0 bo:396 Volume offset:396  Expected:3 Got:2
Mismatch     Block::0 bo:346 Volume offset:346  Expected:1 Got:2
Mismatch     Block::0 bo:397 Volume offset:397  Expected:3 Got:2
Mismatch     Block::0 bo:347 Volume offset:347  Expected:1 Got:2
Mismatch     Block::0 bo:398 Volume offset:398  Expected:3 Got:2
Mismatch     Block::0 bo:348 Volume offset:348  Expected:1 Got:2
Mismatch     Block::0 bo:399 Volume offset:399  Expected:3 Got:2
Mismatch     Block::0 bo:349 Volume offset:349  Expected:1 Got:2
Mismatch     Block::0 bo:400 Volume offset:400  Expected:3 Got:2
Mismatch     Block::0 bo:350 Volume offset:350  Expected:1 Got:2
Mismatch     Block::0 bo:401 Volume offset:401  Expected:3 Got:2
Mismatch     Block::0 bo:351 Volume offset:351  Expected:1 Got:2
Mismatch     Block::0 bo:402 Volume offset:402  Expected:3 Got:2
Mismatch     Block::0 bo:352 Volume offset:352  Expected:1 Got:2
Mismatch     Block::0 bo:403 Volume offset:403  Expected:3 Got:2
Mismatch     Block::0 bo:353 Volume offset:353  Expected:1 Got:2
Mismatch     Block::0 bo:404 Volume offset:404  Expected:3 Got:2
Mismatch     Block::0 bo:354 Volume offset:354  Expected:1 Got:2
Mismatch     Block::0 bo:405 Volume offset:405  Expected:3 Got:2
Mismatch     Block::0 bo:355 Volume offset:355  Expected:1 Got:2
Mismatch     Block::0 bo:406 Volume offset:406  Expected:3 Got:2
Mismatch     Block::0 bo:356 Volume offset:356  Expected:1 Got:2
Mismatch     Block::0 bo:407 Volume offset:407  Expected:3 Got:2
Mismatch     Block::0 bo:357 Volume offset:357  Expected:1 Got:2
Mismatch     Block::0 bo:408 Volume offset:408  Expected:3 Got:2
Mismatch     Block::0 bo:358 Volume offset:358  Expected:1 Got:2
Mismatch     Block::0 bo:409 Volume offset:409  Expected:3 Got:2
Mismatch     Block::0 bo:359 Volume offset:359  Expected:1 Got:2
Mismatch     Block::0 bo:410 Volume offset:410  Expected:3 Got:2
Mismatch     Block::0 bo:360 Volume offset:360  Expected:1 Got:2
Mismatch     Block::0 bo:411 Volume offset:411  Expected:3 Got:2
Mismatch     Block::0 bo:361 Volume offset:361  Expected:1 Got:2
Mismatch     Block::0 bo:412 Volume offset:412  Expected:3 Got:2
Mismatch     Block::0 bo:362 Volume offset:362  Expected:1 Got:2
Mismatch     Block::0 bo:413 Volume offset:413  Expected:3 Got:2
Mismatch     Block::0 bo:363 Volume offset:363  Expected:1 Got:2
Mismatch     Block::0 bo:414 Volume offset:414  Expected:3 Got:2
Mismatch     Block::0 bo:364 Volume offset:364  Expected:1 Got:2
Mismatch     Block::0 bo:415 Volume offset:415  Expected:3 Got:2
Mismatch     Block::0 bo:365 Volume offset:365  Expected:1 Got:2
Mismatch     Block::0 bo:416 Volume offset:416  Expected:3 Got:2
Mismatch     Block::0 bo:366 Volume offset:366  Expected:1 Got:2
Mismatch     Block::0 bo:417 Volume offset:417  Expected:3 Got:2
Mismatch     Block::0 bo:367 Volume offset:367  Expected:1 Got:2
Mismatch     Block::0 bo:418 Volume offset:418  Expected:3 Got:2
Mismatch     Block::0 bo:368 Volume offset:368  Expected:1 Got:2
Mismatch     Block::0 bo:419 Volume offset:419  Expected:3 Got:2
Mismatch     Block::0 bo:369 Volume offset:369  Expected:1 Got:2
Mismatch     Block::0 bo:420 Volume offset:420  Expected:3 Got:2
Mismatch     Block::0 bo:370 Volume offset:370  Expected:1 Got:2
Mismatch     Block::0 bo:421 Volume offset:421  Expected:3 Got:2
Mismatch     Block::0 bo:371 Volume offset:371  Expected:1 Got:2
Mismatch     Block::0 bo:422 Volume offset:422  Expected:3 Got:2
Mismatch     Block::0 bo:372 Volume offset:372  Expected:1 Got:2
Mismatch     Block::0 bo:423 Volume offset:423  Expected:3 Got:2
Mismatch     Block::0 bo:373 Volume offset:373  Expected:1 Got:2
Mismatch     Block::0 bo:424 Volume offset:424  Expected:3 Got:2
Mismatch     Block::0 bo:374 Volume offset:374  Expected:1 Got:2
Mismatch     Block::0 bo:425 Volume offset:425  Expected:3 Got:2
Mismatch     Block::0 bo:375 Volume offset:375  Expected:1 Got:2
Mismatch     Block::0 bo:426 Volume offset:426  Expected:3 Got:2
Mismatch     Block::0 bo:376 Volume offset:376  Expected:1 Got:2
Mismatch     Block::0 bo:427 Volume offset:427  Expected:3 Got:2
Mismatch     Block::0 bo:377 Volume offset:377  Expected:1 Got:2
Mismatch     Block::0 bo:428 Volume offset:428  Expected:3 Got:2
Mismatch     Block::0 bo:378 Volume offset:378  Expected:1 Got:2
Mismatch     Block::0 bo:429 Volume offset:429  Expected:3 Got:2
Mismatch     Block::0 bo:379 Volume offset:379  Expected:1 Got:2
Mismatch     Block::0 bo:430 Volume offset:430  Expected:3 Got:2
Mismatch     Block::0 bo:380 Volume offset:380  Expected:1 Got:2
Mismatch     Block::0 bo:431 Volume offset:431  Expected:3 Got:2
Mismatch     Block::0 bo:381 Volume offset:381  Expected:1 Got:2
Mismatch     Block::0 bo:432 Volume offset:432  Expected:3 Got:2
Mismatch     Block::0 bo:382 Volume offset:382  Expected:1 Got:2
Mismatch     Block::0 bo:433 Volume offset:433  Expected:3 Got:2
Mismatch     Block::0 bo:383 Volume offset:383  Expected:1 Got:2
Mismatch     Block::0 bo:434 Volume offset:434  Expected:3 Got:2
Mismatch     Block::0 bo:384 Volume offset:384  Expected:1 Got:2
Mismatch     Block::0 bo:435 Volume offset:435  Expected:3 Got:2
Mismatch     Block::0 bo:385 Volume offset:385  Expected:1 Got:2
Mismatch     Block::0 bo:436 Volume offset:436  Expected:3 Got:2
Mismatch     Block::0 bo:386 Volume offset:386  Expected:1 Got:2
Mismatch     Block::0 bo:437 Volume offset:437  Expected:3 Got:2
Mismatch     Block::0 bo:387 Volume offset:387  Expected:1 Got:2
Mismatch     Block::0 bo:438 Volume offset:438  Expected:3 Got:2
Mismatch     Block::0 bo:388 Volume offset:388  Expected:1 Got:2
Mismatch     Block::0 bo:439 Volume offset:439  Expected:3 Got:2
Mismatch     Block::0 bo:389 Volume offset:389  Expected:1 Got:2
Mismatch     Block::0 bo:440 Volume offset:440  Expected:3 Got:2
Mismatch     Block::0 bo:390 Volume offset:390  Expected:1 Got:2
Mismatch     Block::0 bo:441 Volume offset:441  Expected:3 Got:2
Mismatch     Block::0 bo:391 Volume offset:391  Expected:1 Got:2
Mismatch     Block::0 bo:442 Volume offset:442  Expected:3 Got:2
Mismatch     Block::0 bo:392 Volume offset:392  Expected:1 Got:2
Mismatch     Block::0 bo:443 Volume offset:443  Expected:3 Got:2
Mismatch     Block::0 bo:393 Volume offset:393  Expected:1 Got:2
Mismatch     Block::0 bo:444 Volume offset:444  Expected:3 Got:2
Mismatch     Block::0 bo:394 Volume offset:394  Expected:1 Got:2
Mismatch     Block::0 bo:445 Volume offset:445  Expected:3 Got:2
Mismatch     Block::0 bo:395 Volume offset:395  Expected:1 Got:2
Mismatch     Block::0 bo:446 Volume offset:446  Expected:3 Got:2
Mismatch     Block::0 bo:396 Volume offset:396  Expected:1 Got:2
Mismatch     Block::0 bo:447 Volume offset:447  Expected:3 Got:2
Mismatch     Block::0 bo:397 Volume offset:397  Expected:1 Got:2
Mismatch     Block::0 bo:448 Volume offset:448  Expected:3 Got:2
Mismatch     Block::0 bo:398 Volume offset:398  Expected:1 Got:2
Mismatch     Block::0 bo:449 Volume offset:449  Expected:3 Got:2
Mismatch     Block::0 bo:399 Volume offset:399  Expected:1 Got:2
Mismatch     Block::0 bo:450 Volume offset:450  Expected:3 Got:2
Mismatch     Block::0 bo:400 Volume offset:400  Expected:1 Got:2
Mismatch     Block::0 bo:451 Volume offset:451  Expected:3 Got:2
Mismatch     Block::0 bo:401 Volume offset:401  Expected:1 Got:2
Mismatch     Block::0 bo:452 Volume offset:452  Expected:3 Got:2
Mismatch     Block::0 bo:402 Volume offset:402  Expected:1 Got:2
Mismatch     Block::0 bo:453 Volume offset:453  Expected:3 Got:2
Mismatch     Block::0 bo:403 Volume offset:403  Expected:1 Got:2
Mismatch     Block::0 bo:454 Volume offset:454  Expected:3 Got:2
Mismatch     Block::0 bo:404 Volume offset:404  Expected:1 Got:2
Mismatch     Block::0 bo:455 Volume offset:455  Expected:3 Got:2
Mismatch     Block::0 bo:405 Volume offset:405  Expected:1 Got:2
Mismatch     Block::0 bo:456 Volume offset:456  Expected:3 Got:2
Mismatch     Block::0 bo:406 Volume offset:406  Expected:1 Got:2
Mismatch     Block::0 bo:457 Volume offset:457  Expected:3 Got:2
Mismatch     Block::0 bo:407 Volume offset:407  Expected:1 Got:2
Mismatch     Block::0 bo:458 Volume offset:458  Expected:3 Got:2
Mismatch     Block::0 bo:408 Volume offset:408  Expected:1 Got:2
Mismatch     Block::0 bo:459 Volume offset:459  Expected:3 Got:2
Mismatch     Block::0 bo:409 Volume offset:409  Expected:1 Got:2
Mismatch     Block::0 bo:460 Volume offset:460  Expected:3 Got:2
Mismatch     Block::0 bo:410 Volume offset:410  Expected:1 Got:2
Mismatch     Block::0 bo:461 Volume offset:461  Expected:3 Got:2
Mismatch     Block::0 bo:411 Volume offset:411  Expected:1 Got:2
Mismatch     Block::0 bo:462 Volume offset:462  Expected:3 Got:2
Mismatch     Block::0 bo:412 Volume offset:412  Expected:1 Got:2
Mismatch     Block::0 bo:463 Volume offset:463  Expected:3 Got:2
Mismatch     Block::0 bo:413 Volume offset:413  Expected:1 Got:2
Mismatch     Block::0 bo:464 Volume offset:464  Expected:3 Got:2
Mismatch     Block::0 bo:414 Volume offset:414  Expected:1 Got:2
Mismatch     Block::0 bo:465 Volume offset:465  Expected:3 Got:2
Mismatch     Block::0 bo:415 Volume offset:415  Expected:1 Got:2
Mismatch     Block::0 bo:466 Volume offset:466  Expected:3 Got:2
Mismatch     Block::0 bo:416 Volume offset:416  Expected:1 Got:2
Mismatch     Block::0 bo:467 Volume offset:467  Expected:3 Got:2
Mismatch     Block::0 bo:417 Volume offset:417  Expected:1 Got:2
Mismatch     Block::0 bo:468 Volume offset:468  Expected:3 Got:2
Mismatch     Block::0 bo:418 Volume offset:418  Expected:1 Got:2
Mismatch     Block::0 bo:469 Volume offset:469  Expected:3 Got:2
Mismatch     Block::0 bo:419 Volume offset:419  Expected:1 Got:2
Mismatch     Block::0 bo:470 Volume offset:470  Expected:3 Got:2
Mismatch     Block::0 bo:420 Volume offset:420  Expected:1 Got:2
Mismatch     Block::0 bo:471 Volume offset:471  Expected:3 Got:2
Mismatch     Block::0 bo:421 Volume offset:421  Expected:1 Got:2
Mismatch     Block::0 bo:472 Volume offset:472  Expected:3 Got:2
Mismatch     Block::0 bo:422 Volume offset:422  Expected:1 Got:2
Mismatch     Block::0 bo:473 Volume offset:473  Expected:3 Got:2
Mismatch     Block::0 bo:423 Volume offset:423  Expected:1 Got:2
Mismatch     Block::0 bo:474 Volume offset:474  Expected:3 Got:2
Mismatch     Block::0 bo:424 Volume offset:424  Expected:1 Got:2
Mismatch     Block::0 bo:475 Volume offset:475  Expected:3 Got:2
Mismatch     Block::0 bo:425 Volume offset:425  Expected:1 Got:2
Mismatch     Block::0 bo:476 Volume offset:476  Expected:3 Got:2
Mismatch     Block::0 bo:426 Volume offset:426  Expected:1 Got:2
Mismatch     Block::0 bo:477 Volume offset:477  Expected:3 Got:2
Mismatch     Block::0 bo:427 Volume offset:427  Expected:1 Got:2
Mismatch     Block::0 bo:478 Volume offset:478  Expected:3 Got:2
Mismatch     Block::0 bo:428 Volume offset:428  Expected:1 Got:2
Mismatch     Block::0 bo:479 Volume offset:479  Expected:3 Got:2
Mismatch     Block::0 bo:429 Volume offset:429  Expected:1 Got:2
Mismatch     Block::0 bo:480 Volume offset:480  Expected:3 Got:2
Mismatch     Block::0 bo:430 Volume offset:430  Expected:1 Got:2
Mismatch     Block::0 bo:481 Volume offset:481  Expected:3 Got:2
Mismatch     Block::0 bo:431 Volume offset:431  Expected:1 Got:2
Mismatch     Block::0 bo:482 Volume offset:482  Expected:3 Got:2
Mismatch     Block::0 bo:432 Volume offset:432  Expected:1 Got:2
Mismatch     Block::0 bo:483 Volume offset:483  Expected:3 Got:2
Mismatch     Block::0 bo:433 Volume offset:433  Expected:1 Got:2
Mismatch     Block::0 bo:484 Volume offset:484  Expected:3 Got:2
Mismatch     Block::0 bo:434 Volume offset:434  Expected:1 Got:2
Mismatch     Block::0 bo:485 Volume offset:485  Expected:3 Got:2
Mismatch     Block::0 bo:435 Volume offset:435  Expected:1 Got:2
Mismatch     Block::0 bo:486 Volume offset:486  Expected:3 Got:2
Mismatch     Block::0 bo:436 Volume offset:436  Expected:1 Got:2
Mismatch     Block::0 bo:487 Volume offset:487  Expected:3 Got:2
Mismatch     Block::0 bo:437 Volume offset:437  Expected:1 Got:2
Mismatch     Block::0 bo:488 Volume offset:488  Expected:3 Got:2
Mismatch     Block::0 bo:438 Volume offset:438  Expected:1 Got:2
Mismatch     Block::0 bo:489 Volume offset:489  Expected:3 Got:2
Mismatch     Block::0 bo:439 Volume offset:439  Expected:1 Got:2
Mismatch     Block::0 bo:490 Volume offset:490  Expected:3 Got:2
Mismatch     Block::0 bo:440 Volume offset:440  Expected:1 Got:2
Mismatch     Block::0 bo:491 Volume offset:491  Expected:3 Got:2
Mismatch     Block::0 bo:441 Volume offset:441  Expected:1 Got:2
Mismatch     Block::0 bo:492 Volume offset:492  Expected:3 Got:2
Mismatch     Block::0 bo:442 Volume offset:442  Expected:1 Got:2
Mismatch     Block::0 bo:493 Volume offset:493  Expected:3 Got:2
Mismatch     Block::0 bo:443 Volume offset:443  Expected:1 Got:2
Mismatch     Block::0 bo:494 Volume offset:494  Expected:3 Got:2
Mismatch     Block::0 bo:444 Volume offset:444  Expected:1 Got:2
Mismatch     Block::0 bo:495 Volume offset:495  Expected:3 Got:2
Mismatch     Block::0 bo:445 Volume offset:445  Expected:1 Got:2
Mismatch     Block::0 bo:496 Volume offset:496  Expected:3 Got:2
Mismatch     Block::0 bo:446 Volume offset:446  Expected:1 Got:2
Mismatch     Block::0 bo:497 Volume offset:497  Expected:3 Got:2
Mismatch     Block::0 bo:447 Volume offset:447  Expected:1 Got:2
Mismatch     Block::0 bo:498 Volume offset:498  Expected:3 Got:2
Mismatch     Block::0 bo:448 Volume offset:448  Expected:1 Got:2
Mismatch     Block::0 bo:499 Volume offset:499  Expected:3 Got:2
Mismatch     Block::0 bo:449 Volume offset:449  Expected:1 Got:2
Mismatch     Block::0 bo:500 Volume offset:500  Expected:3 Got:2
Mismatch     Block::0 bo:450 Volume offset:450  Expected:1 Got:2
Mismatch     Block::0 bo:501 Volume offset:501  Expected:3 Got:2
Mismatch     Block::0 bo:451 Volume offset:451  Expected:1 Got:2
Mismatch     Block::0 bo:502 Volume offset:502  Expected:3 Got:2
Mismatch     Block::0 bo:452 Volume offset:452  Expected:1 Got:2
Mismatch     Block::0 bo:503 Volume offset:503  Expected:3 Got:2
Mismatch     Block::0 bo:453 Volume offset:453  Expected:1 Got:2
Mismatch     Block::0 bo:504 Volume offset:504  Expected:3 Got:2
Mismatch     Block::0 bo:454 Volume offset:454  Expected:1 Got:2
Mismatch     Block::0 bo:505 Volume offset:505  Expected:3 Got:2
Mismatch     Block::0 bo:455 Volume offset:455  Expected:1 Got:2
Mismatch     Block::0 bo:506 Volume offset:506  Expected:3 Got:2
Mismatch     Block::0 bo:456 Volume offset:456  Expected:1 Got:2
Mismatch     Block::0 bo:507 Volume offset:507  Expected:3 Got:2
Mismatch     Block::0 bo:457 Volume offset:457  Expected:1 Got:2
Mismatch     Block::0 bo:508 Volume offset:508  Expected:3 Got:2
Mismatch     Block::0 bo:458 Volume offset:458  Expected:1 Got:2
Mismatch     Block::0 bo:509 Volume offset:509  Expected:3 Got:2
Mismatch     Block::0 bo:459 Volume offset:459  Expected:1 Got:2
Mismatch     Block::0 bo:510 Volume offset:510  Expected:3 Got:2
Mismatch     Block::0 bo:460 Volume offset:460  Expected:1 Got:2
Mismatch     Block::0 bo:511 Volume offset:511  Expected:3 Got:2
Mismatch     Block::0 bo:461 Volume offset:461  Expected:1 Got:2
Mismatch     Block::0 bo:462 Volume offset:462  Expected:1 Got:2

Mismatch     Block::0 bo:463 Volume offset:463  Expected:1 Got:2
Mismatch     Block::0 bo:464 Volume offset:464  Expected:1 Got:2
Mismatch     Block::0 bo:465 Volume offset:465  Expected:1 Got:2
Mismatch     Block::0 bo:466 Volume offset:466  Expected:1 Got:2
Mismatch     Block::0 bo:467 Volume offset:467  Expected:1 Got:2
Mismatch     Block::0 bo:468 Volume offset:468  Expected:1 Got:2
Mismatch     Block::0 bo:469 Volume offset:469  Expected:1 Got:2
test test::test_wl_commit_1024_range_rollover_range ... ok
Mismatch     Block::0 bo:470 Volume offset:470  Expected:1 Got:2
SPEC  v:253  min_av:254 cur_av:1  cm:1022 cc:1025
SPEC  v:254  min_av:254 cur_av:1  cm:1022 cc:1025
new cur is 1022 from min
Adjusting new cur to 1022
new cur is 1024 from cur
Shift 3, v:255 sv:1023 min:1022 cur:1022
Mismatch     Block::99 bo:511 Volume offset:51199  Expected:1 Got:9
Shift 3, v:0 sv:768 min:1022 cur:1022
Shift 3, v:1 sv:769 min:1022 cur:1022
test test::test_wl_commit_1024_range_no_update_below_rollover ... Shift 0, v:1 sv:1 min:2 cur:4
Shift 0, v:2 sv:2 min:2 cur:4
Shift 0, v:3 sv:3 min:2 cur:4
Shift 0, v:4 sv:4 min:2 cur:4
Shift 0, v:5 sv:5 min:2 cur:4
Shift 3, v:2 sv:770 min:1022 cur:1022
ok
Adjusting new cur to 1024
Mismatch     Block::0 bo:471 Volume offset:471  Expected:1 Got:2
SPEC  v:254  min_av:254 cur_av:0  cm:1022 cc:1024
Mismatch     Block::0 bo:472 Volume offset:472  Expected:1 Got:2
new cur is 1022 from min
Mismatch     Block::0 bo:473 Volume offset:473  Expected:1 Got:2
SPEC  v:255  min_av:254 cur_av:0  cm:1022 cc:1024
new cur is 1023 from min
Mismatch     Block::0 bo:474 Volume offset:474  Expected:1 Got:2
SPEC  v:1  min_av:254 cur_av:0  cm:1022 cc:1024
Mismatch     Block::0 bo:475 Volume offset:475  Expected:1 Got:2
SPEC  v:2  min_av:254 cur_av:0  cm:1022 cc:1024
test test::test_read_compare_fail ... Mismatch     Block::0 bo:476 Volume offset:476  Expected:1 Got:2
ok
Mismatch     Block::0 bo:477 Volume offset:477  Expected:1 Got:2
Mismatch     Block::0 bo:478 Volume offset:478  Expected:1 Got:2
Mismatch     Block::0 bo:479 Volume offset:479  Expected:1 Got:2
SPEC  v:2  min_av:254 cur_av:1  cm:254 cc:257
SPEC  v:254  min_av:254 cur_av:1  cm:254 cc:257
new cur is 254 from min
Mismatch     Block::0 bo:480 Volume offset:480  Expected:1 Got:2
Mismatch     Block::0 bo:481 Volume offset:481  Expected:1 Got:2
Mismatch     Block::0 bo:482 Volume offset:482  Expected:1 Got:2
SPEC  v:253  min_av:254 cur_av:1  cm:254 cc:257
SPEC  v:255  min_av:254 cur_av:1  cm:254 cc:257
new cur is 255 from min
SPEC  v:0  min_av:254 cur_av:1  cm:254 cc:257
new cur is 256 from cur
SPEC  v:1  min_av:254 cur_av:1  cm:254 cc:257
test test::test_read_compare_large_fail ... new cur is 257 from cur
SPEC  v:254  min_av:254 cur_av:1  cm:254 cc:257
new cur is 254 from min
SPEC  v:255  min_av:254 cur_av:1  cm:254 cc:257
new cur is 255 from min
SPEC  v:0  min_av:254 cur_av:1  cm:254 cc:257
ok
new cur is 256 from cur
SPEC  v:1  min_av:254 cur_av:1  cm:254 cc:257
new cur is 257 from cur
SPEC  v:2  min_av:254 cur_av:1  cm:254 cc:257
Mismatch     Block::0 bo:483 Volume offset:483  Expected:1 Got:2
Mismatch     Block::0 bo:484 Volume offset:484  Expected:1 Got:2
Mismatch     Block::0 bo:485 Volume offset:485  Expected:1 Got:2
Mismatch     Block::0 bo:486 Volume offset:486  Expected:1 Got:2
Mismatch     Block::0 bo:487 Volume offset:487  Expected:1 Got:2
Mismatch     Block::0 bo:488 Volume offset:488  Expected:1 Got:2
Mismatch     Block::0 bo:489 Volume offset:489  Expected:1 Got:2
Mismatch     Block::0 bo:490 Volume offset:490  Expected:1 Got:2
Mismatch     Block::0 bo:491 Volume offset:491  Expected:1 Got:2
Mismatch     Block::0 bo:492 Volume offset:492  Expected:1 Got:2
Mismatch     Block::0 bo:493 Volume offset:493  Expected:1 Got:2
Mismatch     Block::0 bo:494 Volume offset:494  Expected:1 Got:2
Mismatch     Block::0 bo:495 Volume offset:495  Expected:1 Got:2
Mismatch     Block::0 bo:496 Volume offset:496  Expected:1 Got:2
Mismatch     Block::0 bo:497 Volume offset:497  Expected:1 Got:2
Mismatch     Block::0 bo:498 Volume offset:498  Expected:1 Got:2
Mismatch     Block::0 bo:499 Volume offset:499  Expected:1 Got:2
Mismatch     Block::0 bo:500 Volume offset:500  Expected:1 Got:2
Mismatch     Block::0 bo:501 Volume offset:501  Expected:1 Got:2
Mismatch     Block::0 bo:502 Volume offset:502  Expected:1 Got:2
Mismatch     Block::0 bo:503 Volume offset:503  Expected:1 Got:2
test test::test_wl_commit_range ... Mismatch     Block::0 bo:504 Volume offset:504  Expected:1 Got:2
ok
Mismatch     Block::0 bo:505 Volume offset:505  Expected:1 Got:2
Mismatch     Block::0 bo:506 Volume offset:506  Expected:1 Got:2
SPEC  v:252  min_av:253 cur_av:0  cm:253 cc:256
Mismatch     Block::0 bo:507 Volume offset:507  Expected:1 Got:2
SPEC  v:253  min_av:253 cur_av:0  cm:253 cc:256
Mismatch     Block::0 bo:508 Volume offset:508  Expected:1 Got:2
new cur is 253 from min
Mismatch     Block::0 bo:509 Volume offset:509  Expected:1 Got:2
SPEC  v:254  min_av:253 cur_av:0  cm:253 cc:256
new cur is 254 from min
Mismatch     Block::0 bo:510 Volume offset:510  Expected:1 Got:2
SPEC  v:255  min_av:253 cur_av:0  cm:253 cc:256
test test::test_wl_commit_1024_range_update_rollover_below ... ok
new cur is 255 from min
Mismatch     Block::0 bo:511 Volume offset:511  Expected:1 Got:2
SPEC  v:0  min_av:253 cur_av:0  cm:253 cc:256
new cur is 256 from cur
SPEC  v:254  min_av:255 cur_av:1  cm:255 cc:257
SPEC  v:255  min_av:255 cur_av:1  cm:255 cc:257
new cur is 255 from min
SPEC  v:0  min_av:255 cur_av:1  cm:255 cc:257
new cur is 256 from cur
SPEC  v:1  min_av:253 cur_av:0  cm:253 cc:256
SPEC  v:1  min_av:255 cur_av:1  cm:255 cc:257
new cur is 257 from cur
SPEC  v:2  min_av:255 cur_av:1  cm:255 cc:257
test test::test_wl_commit_1024_range_update_rollover_above ... ok
SPEC  v:253  min_av:254 cur_av:1  cm:254 cc:257
SPEC  v:254  min_av:254 cur_av:1  cm:254 cc:257
new cur is 254 from min
SPEC  v:255  min_av:254 cur_av:1  cm:254 cc:257
new cur is 255 from min
SPEC  v:0  min_av:254 cur_av:1  cm:254 cc:257
new cur is 256 from cur
SPEC  v:1  min_av:254 cur_av:1  cm:254 cc:257
new cur is 257 from cur
SPEC  v:2  min_av:254 cur_av:1  cm:254 cc:257
test test::test_wl_commit_range_no_update_above_rollover ... ok
Shift 0, v:3 sv:3 min:2 cur:4
Update block 1 to 3 (min:2 max:4 res:true)
test test::test_wl_commit_range_no_update_below_rollover ... ok
Shift 0, v:4 sv:4 min:2 cur:4
test test::test_read_compare_fail_under ... ok
Shift 0, v:2 sv:2 min:2 cur:4
Update block 1 to 2 (min:2 max:4 res:true)
test test::test_wl_commit_range_rollover_max_at ... ok
SPEC  v:253  min_av:254 cur_av:1  cm:254 cc:257
SPEC  v:0  min_av:254 cur_av:1  cm:254 cc:257
new cur is 256 from cur
Adjusting new cur to 256
SPEC  v:254  min_av:254 cur_av:0  cm:254 cc:256
new cur is 254 from min
SPEC  v:255  min_av:254 cur_av:0  cm:254 cc:256
new cur is 255 from min
SPEC  v:253  min_av:254 cur_av:1  cm:254 cc:257
SPEC  v:254  min_av:254 cur_av:1  cm:254 cc:257
new cur is 254 from min
Adjusting new cur to 254
Shift 0, v:255 sv:255 min:254 cur:254
Shift 0, v:0 sv:0 min:254 cur:254
Shift 0, v:1 sv:1 min:254 cur:254
Shift 0, v:2 sv:2 min:254 cur:254
test test::test_wl_commit_range_rollover_min_at ... ok
SPEC  v:1  min_av:254 cur_av:0  cm:254 cc:256
SPEC  v:2  min_av:254 cur_av:0  cm:254 cc:256
test test::test_wl_commit_range_rollover_range ... ok
test test::test_wl_commit_range_update ... ok
test test::test_wl_commit_range_update_max ... ok
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:1 Volume offset:513  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:2 Volume offset:514  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:3 Volume offset:515  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:4 Volume offset:516  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:5 Volume offset:517  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:6 Volume offset:518  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:7 Volume offset:519  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:8 Volume offset:520  Expected:4 Got:1
test test::test_wl_commit_range_update_min ... Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:9 Volume offset:521  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:10 Volume offset:522  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:11 Volume offset:523  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:12 Volume offset:524  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:13 Volume offset:525  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:14 Volume offset:526  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:15 Volume offset:527  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:16 Volume offset:528  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:17 Volume offset:529  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:18 Volume offset:530  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:19 Volume offset:531  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:20 Volume offset:532  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:21 Volume offset:533  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:22 Volume offset:534  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:23 Volume offset:535  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:24 Volume offset:536  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:25 Volume offset:537  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:26 Volume offset:538  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:27 Volume offset:539  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:28 Volume offset:540  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:29 Volume offset:541  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:30 Volume offset:542  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:31 Volume offset:543  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:32 Volume offset:544  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:33 Volume offset:545  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:34 Volume offset:546  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:35 Volume offset:547  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:36 Volume offset:548  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:37 Volume offset:549  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:38 Volume offset:550  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:39 Volume offset:551  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:40 Volume offset:552  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:41 Volume offset:553  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:42 Volume offset:554  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:43 Volume offset:555  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:44 Volume offset:556  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:45 Volume offset:557  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:46 Volume offset:558  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:47 Volume offset:559  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:48 Volume offset:560  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:49 Volume offset:561  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:50 Volume offset:562  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:51 Volume offset:563  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:52 Volume offset:564  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:53 Volume offset:565  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:54 Volume offset:566  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:55 Volume offset:567  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:56 Volume offset:568  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:57 Volume offset:569  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:58 Volume offset:570  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:59 Volume offset:571  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:60 Volume offset:572  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:61 Volume offset:573  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:62 Volume offset:574  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:63 Volume offset:575  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:64 Volume offset:576  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:65 Volume offset:577  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:66 Volume offset:578  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:67 Volume offset:579  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:68 Volume offset:580  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:69 Volume offset:581  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:70 Volume offset:582  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:71 Volume offset:583  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:72 Volume offset:584  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:73 Volume offset:585  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:74 Volume offset:586  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:75 Volume offset:587  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:76 Volume offset:588  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:77 Volume offset:589  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:78 Volume offset:590  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:79 Volume offset:591  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:80 Volume offset:592  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:81 Volume offset:593  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:82 Volume offset:594  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:83 Volume offset:595  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:84 Volume offset:596  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:85 Volume offset:597  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:86 Volume offset:598  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:87 Volume offset:599  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:88 Volume offset:600  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:89 Volume offset:601  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:90 Volume offset:602  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:91 Volume offset:603  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:92 Volume offset:604  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:93 Volume offset:605  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:94 Volume offset:606  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:95 Volume offset:607  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:96 Volume offset:608  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:97 Volume offset:609  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:98 Volume offset:610  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:99 Volume offset:611  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:100 Volume offset:612  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:101 Volume offset:613  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:102 Volume offset:614  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:103 Volume offset:615  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:104 Volume offset:616  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:105 Volume offset:617  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:106 Volume offset:618  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:107 Volume offset:619  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:108 Volume offset:620  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:109 Volume offset:621  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:110 Volume offset:622  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:111 Volume offset:623  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:112 Volume offset:624  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:113 Volume offset:625  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:114 Volume offset:626  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:115 Volume offset:627  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:116 Volume offset:628  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:117 Volume offset:629  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:118 Volume offset:630  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:119 Volume offset:631  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:120 Volume offset:632  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:121 Volume offset:633  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:122 Volume offset:634  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:123 Volume offset:635  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:124 Volume offset:636  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:125 Volume offset:637  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:126 Volume offset:638  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:127 Volume offset:639  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
okOut of Range Block::1 bo:128 Volume offset:640  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:129 Volume offset:641  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:130 Volume offset:642  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:131 Volume offset:643  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:132 Volume offset:644  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:133 Volume offset:645  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:134 Volume offset:646  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:135 Volume offset:647  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:136 Volume offset:648  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:137 Volume offset:649  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:138 Volume offset:650  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:139 Volume offset:651  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:140 Volume offset:652  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:141 Volume offset:653  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:142 Volume offset:654  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:143 Volume offset:655  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:144 Volume offset:656  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:145 Volume offset:657  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:146 Volume offset:658  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:147 Volume offset:659  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:148 Volume offset:660  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:149 Volume offset:661  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:150 Volume offset:662  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:151 Volume offset:663  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:152 Volume offset:664  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:153 Volume offset:665  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:154 Volume offset:666  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:155 Volume offset:667  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:156 Volume offset:668  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:157 Volume offset:669  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:158 Volume offset:670  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:159 Volume offset:671  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:160 Volume offset:672  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:161 Volume offset:673  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:162 Volume offset:674  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:163 Volume offset:675  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:164 Volume offset:676  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:165 Volume offset:677  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4

Out of Range Block::1 bo:166 Volume offset:678  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:167 Volume offset:679  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:168 Volume offset:680  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:169 Volume offset:681  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:170 Volume offset:682  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:171 Volume offset:683  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:172 Volume offset:684  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:173 Volume offset:685  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:174 Volume offset:686  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:175 Volume offset:687  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:176 Volume offset:688  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:177 Volume offset:689  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:178 Volume offset:690  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:179 Volume offset:691  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:180 Volume offset:692  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:181 Volume offset:693  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:182 Volume offset:694  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:183 Volume offset:695  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:184 Volume offset:696  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:185 Volume offset:697  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:186 Volume offset:698  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:187 Volume offset:699  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:188 Volume offset:700  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:189 Volume offset:701  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:190 Volume offset:702  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:191 Volume offset:703  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:192 Volume offset:704  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:193 Volume offset:705  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:194 Volume offset:706  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:195 Volume offset:707  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:196 Volume offset:708  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:197 Volume offset:709  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:198 Volume offset:710  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:199 Volume offset:711  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:200 Volume offset:712  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:201 Volume offset:713  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:202 Volume offset:714  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:203 Volume offset:715  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:204 Volume offset:716  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:205 Volume offset:717  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:206 Volume offset:718  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:207 Volume offset:719  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:208 Volume offset:720  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:209 Volume offset:721  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:210 Volume offset:722  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:211 Volume offset:723  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:212 Volume offset:724  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:213 Volume offset:725  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:214 Volume offset:726  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:215 Volume offset:727  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:216 Volume offset:728  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:217 Volume offset:729  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:218 Volume offset:730  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:219 Volume offset:731  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:220 Volume offset:732  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:221 Volume offset:733  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:222 Volume offset:734  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:223 Volume offset:735  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:224 Volume offset:736  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:225 Volume offset:737  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:226 Volume offset:738  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:227 Volume offset:739  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:228 Volume offset:740  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:229 Volume offset:741  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:230 Volume offset:742  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:231 Volume offset:743  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:232 Volume offset:744  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:233 Volume offset:745  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:234 Volume offset:746  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:235 Volume offset:747  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:236 Volume offset:748  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:237 Volume offset:749  Expected:4 Got:1
test test::test_wl_commit_range_update_rollover_below ... Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:238 Volume offset:750  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:239 Volume offset:751  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:240 Volume offset:752  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:241 Volume offset:753  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:242 Volume offset:754  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:243 Volume offset:755  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:244 Volume offset:756  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:245 Volume offset:757  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:246 Volume offset:758  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:247 Volume offset:759  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:248 Volume offset:760  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:249 Volume offset:761  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:250 Volume offset:762  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:251 Volume offset:763  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:252 Volume offset:764  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:253 Volume offset:765  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:254 Volume offset:766  Expected:4 Got:1
okShift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:255 Volume offset:767  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:256 Volume offset:768  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:257 Volume offset:769  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:258 Volume offset:770  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:259 Volume offset:771  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:260 Volume offset:772  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:261 Volume offset:773  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:262 Volume offset:774  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:263 Volume offset:775  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:264 Volume offset:776  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:265 Volume offset:777  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:266 Volume offset:778  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:267 Volume offset:779  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:268 Volume offset:780  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:269 Volume offset:781  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:270 Volume offset:782  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:271 Volume offset:783  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:272 Volume offset:784  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:273 Volume offset:785  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:274 Volume offset:786  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:275 Volume offset:787  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:276 Volume offset:788  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:277 Volume offset:789  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:278 Volume offset:790  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:279 Volume offset:791  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:280 Volume offset:792  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:281 Volume offset:793  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:282 Volume offset:794  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:283 Volume offset:795  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:284 Volume offset:796  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:285 Volume offset:797  Expected:4 Got:1

Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:286 Volume offset:798  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:287 Volume offset:799  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:288 Volume offset:800  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:289 Volume offset:801  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:290 Volume offset:802  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:291 Volume offset:803  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:292 Volume offset:804  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:293 Volume offset:805  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:294 Volume offset:806  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:295 Volume offset:807  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:296 Volume offset:808  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:297 Volume offset:809  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:298 Volume offset:810  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:299 Volume offset:811  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:300 Volume offset:812  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:301 Volume offset:813  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:302 Volume offset:814  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:303 Volume offset:815  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:304 Volume offset:816  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:305 Volume offset:817  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:306 Volume offset:818  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:307 Volume offset:819  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:308 Volume offset:820  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:309 Volume offset:821  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:310 Volume offset:822  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:311 Volume offset:823  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:312 Volume offset:824  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:313 Volume offset:825  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:314 Volume offset:826  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:315 Volume offset:827  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:316 Volume offset:828  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:317 Volume offset:829  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:318 Volume offset:830  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:319 Volume offset:831  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:320 Volume offset:832  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:321 Volume offset:833  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:322 Volume offset:834  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:323 Volume offset:835  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:324 Volume offset:836  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:325 Volume offset:837  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:326 Volume offset:838  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:327 Volume offset:839  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:328 Volume offset:840  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:329 Volume offset:841  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:330 Volume offset:842  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:331 Volume offset:843  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:332 Volume offset:844  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:333 Volume offset:845  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:334 Volume offset:846  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:335 Volume offset:847  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:336 Volume offset:848  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:337 Volume offset:849  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:338 Volume offset:850  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:339 Volume offset:851  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:340 Volume offset:852  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:341 Volume offset:853  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:342 Volume offset:854  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:343 Volume offset:855  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:344 Volume offset:856  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:345 Volume offset:857  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:346 Volume offset:858  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:347 Volume offset:859  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:348 Volume offset:860  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:349 Volume offset:861  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:350 Volume offset:862  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:351 Volume offset:863  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:352 Volume offset:864  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:353 Volume offset:865  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:354 Volume offset:866  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:355 Volume offset:867  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:356 Volume offset:868  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:357 Volume offset:869  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:358 Volume offset:870  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:359 Volume offset:871  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
test test::test_wl_commit_range_update_rollover_above ... Out of Range Block::1 bo:360 Volume offset:872  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:361 Volume offset:873  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:362 Volume offset:874  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:363 Volume offset:875  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:364 Volume offset:876  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:365 Volume offset:877  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:366 Volume offset:878  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:367 Volume offset:879  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:368 Volume offset:880  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:369 Volume offset:881  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:370 Volume offset:882  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:371 Volume offset:883  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:372 Volume offset:884  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:373 Volume offset:885  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:374 Volume offset:886  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:375 Volume offset:887  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:376 Volume offset:888  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:377 Volume offset:889  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:378 Volume offset:890  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:379 Volume offset:891  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:380 Volume offset:892  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:381 Volume offset:893  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:382 Volume offset:894  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:383 Volume offset:895  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:384 Volume offset:896  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:385 Volume offset:897  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:386 Volume offset:898  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:387 Volume offset:899  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:388 Volume offset:900  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:389 Volume offset:901  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
okOut of Range Block::1 bo:390 Volume offset:902  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:391 Volume offset:903  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:392 Volume offset:904  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:393 Volume offset:905  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:394 Volume offset:906  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:395 Volume offset:907  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:396 Volume offset:908  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:397 Volume offset:909  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:398 Volume offset:910  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:399 Volume offset:911  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:400 Volume offset:912  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:401 Volume offset:913  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:402 Volume offset:914  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:403 Volume offset:915  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:404 Volume offset:916  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:405 Volume offset:917  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:406 Volume offset:918  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:407 Volume offset:919  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:408 Volume offset:920  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:409 Volume offset:921  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:410 Volume offset:922  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:411 Volume offset:923  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:412 Volume offset:924  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:413 Volume offset:925  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:414 Volume offset:926  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:415 Volume offset:927  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:416 Volume offset:928  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4

Out of Range Block::1 bo:417 Volume offset:929  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:418 Volume offset:930  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:419 Volume offset:931  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:420 Volume offset:932  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:421 Volume offset:933  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:422 Volume offset:934  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:423 Volume offset:935  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:424 Volume offset:936  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:425 Volume offset:937  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:426 Volume offset:938  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:427 Volume offset:939  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:428 Volume offset:940  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:429 Volume offset:941  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:430 Volume offset:942  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:431 Volume offset:943  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:432 Volume offset:944  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:433 Volume offset:945  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:434 Volume offset:946  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:435 Volume offset:947  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:436 Volume offset:948  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:437 Volume offset:949  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:438 Volume offset:950  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:439 Volume offset:951  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:440 Volume offset:952  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:441 Volume offset:953  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:442 Volume offset:954  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:443 Volume offset:955  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:444 Volume offset:956  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:445 Volume offset:957  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:446 Volume offset:958  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:447 Volume offset:959  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:448 Volume offset:960  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:449 Volume offset:961  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:450 Volume offset:962  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:451 Volume offset:963  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:452 Volume offset:964  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:453 Volume offset:965  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:454 Volume offset:966  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:455 Volume offset:967  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:456 Volume offset:968  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:457 Volume offset:969  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:458 Volume offset:970  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:459 Volume offset:971  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:460 Volume offset:972  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:461 Volume offset:973  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:462 Volume offset:974  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:463 Volume offset:975  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:464 Volume offset:976  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:465 Volume offset:977  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:466 Volume offset:978  Expected:4 Got:1
test test::test_read_compare_large ... Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:467 Volume offset:979  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:468 Volume offset:980  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:469 Volume offset:981  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:470 Volume offset:982  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:471 Volume offset:983  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:472 Volume offset:984  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:473 Volume offset:985  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:474 Volume offset:986  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:475 Volume offset:987  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:476 Volume offset:988  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:477 Volume offset:989  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:478 Volume offset:990  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:479 Volume offset:991  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:480 Volume offset:992  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:481 Volume offset:993  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:482 Volume offset:994  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:483 Volume offset:995  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:484 Volume offset:996  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:485 Volume offset:997  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:486 Volume offset:998  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:487 Volume offset:999  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:488 Volume offset:1000  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:489 Volume offset:1001  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:490 Volume offset:1002  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:491 Volume offset:1003  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:492 Volume offset:1004  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:493 Volume offset:1005  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:494 Volume offset:1006  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:495 Volume offset:1007  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:496 Volume offset:1008  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:497 Volume offset:1009  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:498 Volume offset:1010  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:499 Volume offset:1011  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:500 Volume offset:1012  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:501 Volume offset:1013  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:502 Volume offset:1014  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:503 Volume offset:1015  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:504 Volume offset:1016  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:505 Volume offset:1017  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
okOut of Range Block::1 bo:506 Volume offset:1018  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:507 Volume offset:1019  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:508 Volume offset:1020  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:509 Volume offset:1021  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:510 Volume offset:1022  Expected:4 Got:1
Shift 0, v:1 sv:1 min:2 cur:4
Out of Range Block::1 bo:511 Volume offset:1023  Expected:4 Got:1

Shift 0, v:2 sv:2 min:2 cur:4
Update block 1 to 2 (min:2 max:4 res:true)
In Range   Block::1 bo:1 Volume offset:513  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
test test::test_wl_empty ... In Range   Block::1 bo:2 Volume offset:514  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:3 Volume offset:515  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:4 Volume offset:516  Expected:4 Got:2
okShift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:5 Volume offset:517  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:6 Volume offset:518  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:7 Volume offset:519  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:8 Volume offset:520  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:9 Volume offset:521  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:10 Volume offset:522  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:11 Volume offset:523  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:12 Volume offset:524  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:13 Volume offset:525  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:14 Volume offset:526  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:15 Volume offset:527  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:16 Volume offset:528  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:17 Volume offset:529  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:18 Volume offset:530  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:19 Volume offset:531  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:20 Volume offset:532  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:21 Volume offset:533  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:22 Volume offset:534  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:23 Volume offset:535  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:24 Volume offset:536  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:25 Volume offset:537  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:26 Volume offset:538  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:27 Volume offset:539  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:28 Volume offset:540  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:29 Volume offset:541  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:30 Volume offset:542  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:31 Volume offset:543  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:32 Volume offset:544  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:33 Volume offset:545  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:34 Volume offset:546  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:35 Volume offset:547  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:36 Volume offset:548  Expected:4 Got:2

Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:37 Volume offset:549  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:38 Volume offset:550  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:39 Volume offset:551  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:40 Volume offset:552  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:41 Volume offset:553  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:42 Volume offset:554  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:43 Volume offset:555  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:44 Volume offset:556  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:45 Volume offset:557  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:46 Volume offset:558  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:47 Volume offset:559  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:48 Volume offset:560  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:49 Volume offset:561  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:50 Volume offset:562  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:51 Volume offset:563  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:52 Volume offset:564  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:53 Volume offset:565  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
test test::test_wl_is_zero ... In Range   Block::1 bo:54 Volume offset:566  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:55 Volume offset:567  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:56 Volume offset:568  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
okIn Range   Block::1 bo:57 Volume offset:569  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:58 Volume offset:570  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:59 Volume offset:571  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:60 Volume offset:572  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:61 Volume offset:573  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:62 Volume offset:574  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:63 Volume offset:575  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:64 Volume offset:576  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:65 Volume offset:577  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:66 Volume offset:578  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:67 Volume offset:579  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:68 Volume offset:580  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:69 Volume offset:581  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:70 Volume offset:582  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:71 Volume offset:583  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:72 Volume offset:584  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:73 Volume offset:585  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:74 Volume offset:586  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:75 Volume offset:587  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:76 Volume offset:588  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:77 Volume offset:589  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:78 Volume offset:590  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:79 Volume offset:591  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:80 Volume offset:592  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:81 Volume offset:593  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:82 Volume offset:594  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:83 Volume offset:595  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:84 Volume offset:596  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:85 Volume offset:597  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:86 Volume offset:598  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:87 Volume offset:599  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:88 Volume offset:600  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:89 Volume offset:601  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:90 Volume offset:602  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:91 Volume offset:603  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:92 Volume offset:604  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:93 Volume offset:605  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:94 Volume offset:606  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:95 Volume offset:607  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:96 Volume offset:608  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:97 Volume offset:609  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2

In Range   Block::1 bo:98 Volume offset:610  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:99 Volume offset:611  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:100 Volume offset:612  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:101 Volume offset:613  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:102 Volume offset:614  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:103 Volume offset:615  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:104 Volume offset:616  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:105 Volume offset:617  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:106 Volume offset:618  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
test test::test_wl_set ... In Range   Block::1 bo:107 Volume offset:619  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:108 Volume offset:620  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:109 Volume offset:621  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:110 Volume offset:622  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:111 Volume offset:623  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:112 Volume offset:624  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:113 Volume offset:625  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:114 Volume offset:626  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:115 Volume offset:627  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:116 Volume offset:628  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:117 Volume offset:629  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:118 Volume offset:630  Expected:4 Got:2
okShift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:119 Volume offset:631  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:120 Volume offset:632  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:121 Volume offset:633  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:122 Volume offset:634  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:123 Volume offset:635  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:124 Volume offset:636  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:125 Volume offset:637  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:126 Volume offset:638  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:127 Volume offset:639  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:128 Volume offset:640  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:129 Volume offset:641  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:130 Volume offset:642  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:131 Volume offset:643  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:132 Volume offset:644  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:133 Volume offset:645  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:134 Volume offset:646  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:135 Volume offset:647  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:136 Volume offset:648  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:137 Volume offset:649  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:138 Volume offset:650  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:139 Volume offset:651  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:140 Volume offset:652  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:141 Volume offset:653  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:142 Volume offset:654  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:143 Volume offset:655  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:144 Volume offset:656  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:145 Volume offset:657  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:146 Volume offset:658  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:147 Volume offset:659  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:148 Volume offset:660  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:149 Volume offset:661  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:150 Volume offset:662  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:151 Volume offset:663  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:152 Volume offset:664  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:153 Volume offset:665  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:154 Volume offset:666  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:155 Volume offset:667  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:156 Volume offset:668  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:157 Volume offset:669  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:158 Volume offset:670  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:159 Volume offset:671  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:160 Volume offset:672  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:161 Volume offset:673  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:162 Volume offset:674  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:163 Volume offset:675  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:164 Volume offset:676  Expected:4 Got:2

Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:165 Volume offset:677  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:166 Volume offset:678  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:167 Volume offset:679  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:168 Volume offset:680  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:169 Volume offset:681  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:170 Volume offset:682  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:171 Volume offset:683  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:172 Volume offset:684  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:173 Volume offset:685  Expected:4 Got:2
test test::test_wl_update ... Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:174 Volume offset:686  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:175 Volume offset:687  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:176 Volume offset:688  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:177 Volume offset:689  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:178 Volume offset:690  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:179 Volume offset:691  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:180 Volume offset:692  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:181 Volume offset:693  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:182 Volume offset:694  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:183 Volume offset:695  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:184 Volume offset:696  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:185 Volume offset:697  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:186 Volume offset:698  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:187 Volume offset:699  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:188 Volume offset:700  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:189 Volume offset:701  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:190 Volume offset:702  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:191 Volume offset:703  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:192 Volume offset:704  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:193 Volume offset:705  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:194 Volume offset:706  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:195 Volume offset:707  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:196 Volume offset:708  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:197 Volume offset:709  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:198 Volume offset:710  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:199 Volume offset:711  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:200 Volume offset:712  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:201 Volume offset:713  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:202 Volume offset:714  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:203 Volume offset:715  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:204 Volume offset:716  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:205 Volume offset:717  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:206 Volume offset:718  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:207 Volume offset:719  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:208 Volume offset:720  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:209 Volume offset:721  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:210 Volume offset:722  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:211 Volume offset:723  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:212 Volume offset:724  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:213 Volume offset:725  Expected:4 Got:2
okShift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:214 Volume offset:726  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:215 Volume offset:727  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:216 Volume offset:728  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:217 Volume offset:729  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:218 Volume offset:730  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:219 Volume offset:731  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:220 Volume offset:732  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:221 Volume offset:733  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:222 Volume offset:734  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:223 Volume offset:735  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:224 Volume offset:736  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:225 Volume offset:737  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:226 Volume offset:738  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:227 Volume offset:739  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:228 Volume offset:740  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:229 Volume offset:741  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:230 Volume offset:742  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:231 Volume offset:743  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:232 Volume offset:744  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:233 Volume offset:745  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:234 Volume offset:746  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:235 Volume offset:747  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:236 Volume offset:748  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:237 Volume offset:749  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:238 Volume offset:750  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:239 Volume offset:751  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:240 Volume offset:752  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:241 Volume offset:753  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:242 Volume offset:754  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:243 Volume offset:755  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:244 Volume offset:756  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:245 Volume offset:757  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:246 Volume offset:758  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:247 Volume offset:759  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:248 Volume offset:760  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:249 Volume offset:761  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:250 Volume offset:762  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:251 Volume offset:763  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:252 Volume offset:764  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:253 Volume offset:765  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:254 Volume offset:766  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:255 Volume offset:767  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:256 Volume offset:768  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:257 Volume offset:769  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:258 Volume offset:770  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:259 Volume offset:771  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:260 Volume offset:772  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:261 Volume offset:773  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:262 Volume offset:774  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:263 Volume offset:775  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:264 Volume offset:776  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:265 Volume offset:777  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:266 Volume offset:778  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:267 Volume offset:779  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:268 Volume offset:780  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2

In Range   Block::1 bo:269 Volume offset:781  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:270 Volume offset:782  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:271 Volume offset:783  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:272 Volume offset:784  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:273 Volume offset:785  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:274 Volume offset:786  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:275 Volume offset:787  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:276 Volume offset:788  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:277 Volume offset:789  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:278 Volume offset:790  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:279 Volume offset:791  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:280 Volume offset:792  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:281 Volume offset:793  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:282 Volume offset:794  Expected:4 Got:2
test test::test_wl_update_commit ... Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:283 Volume offset:795  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:284 Volume offset:796  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:285 Volume offset:797  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:286 Volume offset:798  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:287 Volume offset:799  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:288 Volume offset:800  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:289 Volume offset:801  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:290 Volume offset:802  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:291 Volume offset:803  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:292 Volume offset:804  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:293 Volume offset:805  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:294 Volume offset:806  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:295 Volume offset:807  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:296 Volume offset:808  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:297 Volume offset:809  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:298 Volume offset:810  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:299 Volume offset:811  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:300 Volume offset:812  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:301 Volume offset:813  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:302 Volume offset:814  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:303 Volume offset:815  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:304 Volume offset:816  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:305 Volume offset:817  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:306 Volume offset:818  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:307 Volume offset:819  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:308 Volume offset:820  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:309 Volume offset:821  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:310 Volume offset:822  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:311 Volume offset:823  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:312 Volume offset:824  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:313 Volume offset:825  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:314 Volume offset:826  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:315 Volume offset:827  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:316 Volume offset:828  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:317 Volume offset:829  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:318 Volume offset:830  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:319 Volume offset:831  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:320 Volume offset:832  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:321 Volume offset:833  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:322 Volume offset:834  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:323 Volume offset:835  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:324 Volume offset:836  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:325 Volume offset:837  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:326 Volume offset:838  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:327 Volume offset:839  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:328 Volume offset:840  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:329 Volume offset:841  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:330 Volume offset:842  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:331 Volume offset:843  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:332 Volume offset:844  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:333 Volume offset:845  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:334 Volume offset:846  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:335 Volume offset:847  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:336 Volume offset:848  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:337 Volume offset:849  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:338 Volume offset:850  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:339 Volume offset:851  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:340 Volume offset:852  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:341 Volume offset:853  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:342 Volume offset:854  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:343 Volume offset:855  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:344 Volume offset:856  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:345 Volume offset:857  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:346 Volume offset:858  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:347 Volume offset:859  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:348 Volume offset:860  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:349 Volume offset:861  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:350 Volume offset:862  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:351 Volume offset:863  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:352 Volume offset:864  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:353 Volume offset:865  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:354 Volume offset:866  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:355 Volume offset:867  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:356 Volume offset:868  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:357 Volume offset:869  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:358 Volume offset:870  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:359 Volume offset:871  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:360 Volume offset:872  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:361 Volume offset:873  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:362 Volume offset:874  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:363 Volume offset:875  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:364 Volume offset:876  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:365 Volume offset:877  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:366 Volume offset:878  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:367 Volume offset:879  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:368 Volume offset:880  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:369 Volume offset:881  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:370 Volume offset:882  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:371 Volume offset:883  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:372 Volume offset:884  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:373 Volume offset:885  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
okIn Range   Block::1 bo:374 Volume offset:886  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:375 Volume offset:887  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:376 Volume offset:888  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:377 Volume offset:889  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:378 Volume offset:890  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:379 Volume offset:891  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:380 Volume offset:892  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:381 Volume offset:893  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:382 Volume offset:894  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:383 Volume offset:895  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:384 Volume offset:896  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:385 Volume offset:897  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:386 Volume offset:898  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:387 Volume offset:899  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:388 Volume offset:900  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:389 Volume offset:901  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:390 Volume offset:902  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:391 Volume offset:903  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:392 Volume offset:904  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:393 Volume offset:905  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:394 Volume offset:906  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:395 Volume offset:907  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:396 Volume offset:908  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:397 Volume offset:909  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:398 Volume offset:910  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:399 Volume offset:911  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:400 Volume offset:912  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:401 Volume offset:913  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:402 Volume offset:914  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:403 Volume offset:915  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:404 Volume offset:916  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:405 Volume offset:917  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:406 Volume offset:918  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:407 Volume offset:919  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:408 Volume offset:920  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:409 Volume offset:921  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:410 Volume offset:922  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:411 Volume offset:923  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:412 Volume offset:924  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:413 Volume offset:925  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:414 Volume offset:926  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:415 Volume offset:927  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:416 Volume offset:928  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:417 Volume offset:929  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:418 Volume offset:930  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:419 Volume offset:931  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:420 Volume offset:932  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:421 Volume offset:933  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:422 Volume offset:934  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:423 Volume offset:935  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:424 Volume offset:936  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:425 Volume offset:937  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:426 Volume offset:938  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:427 Volume offset:939  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:428 Volume offset:940  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:429 Volume offset:941  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:430 Volume offset:942  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:431 Volume offset:943  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:432 Volume offset:944  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:433 Volume offset:945  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:434 Volume offset:946  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:435 Volume offset:947  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:436 Volume offset:948  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:437 Volume offset:949  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:438 Volume offset:950  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:439 Volume offset:951  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:440 Volume offset:952  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:441 Volume offset:953  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:442 Volume offset:954  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:443 Volume offset:955  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:444 Volume offset:956  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:445 Volume offset:957  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:446 Volume offset:958  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:447 Volume offset:959  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:448 Volume offset:960  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:449 Volume offset:961  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:450 Volume offset:962  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:451 Volume offset:963  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:452 Volume offset:964  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:453 Volume offset:965  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:454 Volume offset:966  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:455 Volume offset:967  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:456 Volume offset:968  Expected:4 Got:2

Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:457 Volume offset:969  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:458 Volume offset:970  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:459 Volume offset:971  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:460 Volume offset:972  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:461 Volume offset:973  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:462 Volume offset:974  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:463 Volume offset:975  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:464 Volume offset:976  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:465 Volume offset:977  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:466 Volume offset:978  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:467 Volume offset:979  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:468 Volume offset:980  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:469 Volume offset:981  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:470 Volume offset:982  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:471 Volume offset:983  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:472 Volume offset:984  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:473 Volume offset:985  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:474 Volume offset:986  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:475 Volume offset:987  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:476 Volume offset:988  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:477 Volume offset:989  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:478 Volume offset:990  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:479 Volume offset:991  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:480 Volume offset:992  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:481 Volume offset:993  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:482 Volume offset:994  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:483 Volume offset:995  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:484 Volume offset:996  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:485 Volume offset:997  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:486 Volume offset:998  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:487 Volume offset:999  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:488 Volume offset:1000  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:489 Volume offset:1001  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:490 Volume offset:1002  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:491 Volume offset:1003  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:492 Volume offset:1004  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:493 Volume offset:1005  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:494 Volume offset:1006  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:495 Volume offset:1007  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:496 Volume offset:1008  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:497 Volume offset:1009  Expected:4 Got:2
test test::test_wl_update_commit_2 ... Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:498 Volume offset:1010  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:499 Volume offset:1011  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:500 Volume offset:1012  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:501 Volume offset:1013  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:502 Volume offset:1014  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:503 Volume offset:1015  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:504 Volume offset:1016  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:505 Volume offset:1017  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:506 Volume offset:1018  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:507 Volume offset:1019  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:508 Volume offset:1020  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:509 Volume offset:1021  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:510 Volume offset:1022  Expected:4 Got:2
Shift 0, v:2 sv:2 min:2 cur:2
In Range   Block::1 bo:511 Volume offset:1023  Expected:4 Got:2
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:1 Volume offset:513  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:2 Volume offset:514  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:3 Volume offset:515  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:4 Volume offset:516  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:5 Volume offset:517  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:6 Volume offset:518  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:7 Volume offset:519  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:8 Volume offset:520  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:9 Volume offset:521  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:10 Volume offset:522  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:11 Volume offset:523  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:12 Volume offset:524  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:13 Volume offset:525  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:14 Volume offset:526  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:15 Volume offset:527  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:16 Volume offset:528  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:17 Volume offset:529  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:18 Volume offset:530  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:19 Volume offset:531  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:20 Volume offset:532  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:21 Volume offset:533  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:22 Volume offset:534  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:23 Volume offset:535  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:24 Volume offset:536  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:25 Volume offset:537  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:26 Volume offset:538  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:27 Volume offset:539  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:28 Volume offset:540  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:29 Volume offset:541  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:30 Volume offset:542  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:31 Volume offset:543  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:32 Volume offset:544  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:33 Volume offset:545  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:34 Volume offset:546  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:35 Volume offset:547  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:36 Volume offset:548  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:37 Volume offset:549  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:38 Volume offset:550  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:39 Volume offset:551  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:40 Volume offset:552  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:41 Volume offset:553  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:42 Volume offset:554  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:43 Volume offset:555  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:44 Volume offset:556  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:45 Volume offset:557  Expected:2 Got:4
okShift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:46 Volume offset:558  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:47 Volume offset:559  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:48 Volume offset:560  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:49 Volume offset:561  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:50 Volume offset:562  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:51 Volume offset:563  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:52 Volume offset:564  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2

Out of Range Block::1 bo:53 Volume offset:565  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:54 Volume offset:566  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:55 Volume offset:567  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:56 Volume offset:568  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:57 Volume offset:569  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:58 Volume offset:570  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:59 Volume offset:571  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:60 Volume offset:572  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:61 Volume offset:573  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:62 Volume offset:574  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:63 Volume offset:575  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:64 Volume offset:576  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:65 Volume offset:577  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:66 Volume offset:578  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:67 Volume offset:579  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:68 Volume offset:580  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:69 Volume offset:581  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:70 Volume offset:582  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:71 Volume offset:583  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:72 Volume offset:584  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:73 Volume offset:585  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:74 Volume offset:586  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:75 Volume offset:587  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:76 Volume offset:588  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:77 Volume offset:589  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:78 Volume offset:590  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:79 Volume offset:591  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:80 Volume offset:592  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:81 Volume offset:593  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:82 Volume offset:594  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:83 Volume offset:595  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:84 Volume offset:596  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:85 Volume offset:597  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:86 Volume offset:598  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:87 Volume offset:599  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:88 Volume offset:600  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:89 Volume offset:601  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:90 Volume offset:602  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:91 Volume offset:603  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:92 Volume offset:604  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:93 Volume offset:605  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:94 Volume offset:606  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:95 Volume offset:607  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:96 Volume offset:608  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:97 Volume offset:609  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:98 Volume offset:610  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:99 Volume offset:611  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:100 Volume offset:612  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:101 Volume offset:613  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:102 Volume offset:614  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:103 Volume offset:615  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:104 Volume offset:616  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:105 Volume offset:617  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:106 Volume offset:618  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:107 Volume offset:619  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:108 Volume offset:620  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:109 Volume offset:621  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:110 Volume offset:622  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:111 Volume offset:623  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:112 Volume offset:624  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:113 Volume offset:625  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:114 Volume offset:626  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:115 Volume offset:627  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:116 Volume offset:628  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:117 Volume offset:629  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:118 Volume offset:630  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:119 Volume offset:631  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:120 Volume offset:632  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:121 Volume offset:633  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:122 Volume offset:634  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:123 Volume offset:635  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:124 Volume offset:636  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:125 Volume offset:637  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:126 Volume offset:638  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:127 Volume offset:639  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:128 Volume offset:640  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:129 Volume offset:641  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:130 Volume offset:642  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:131 Volume offset:643  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:132 Volume offset:644  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:133 Volume offset:645  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:134 Volume offset:646  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:135 Volume offset:647  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:136 Volume offset:648  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:137 Volume offset:649  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:138 Volume offset:650  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:139 Volume offset:651  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:140 Volume offset:652  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:141 Volume offset:653  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:142 Volume offset:654  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:143 Volume offset:655  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:144 Volume offset:656  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:145 Volume offset:657  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:146 Volume offset:658  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:147 Volume offset:659  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:148 Volume offset:660  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:149 Volume offset:661  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:150 Volume offset:662  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:151 Volume offset:663  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:152 Volume offset:664  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:153 Volume offset:665  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:154 Volume offset:666  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:155 Volume offset:667  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
test test::test_wl_update_commit_rollover ... Out of Range Block::1 bo:156 Volume offset:668  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:157 Volume offset:669  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:158 Volume offset:670  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:159 Volume offset:671  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:160 Volume offset:672  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:161 Volume offset:673  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:162 Volume offset:674  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:163 Volume offset:675  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:164 Volume offset:676  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:165 Volume offset:677  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:166 Volume offset:678  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:167 Volume offset:679  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:168 Volume offset:680  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:169 Volume offset:681  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:170 Volume offset:682  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:171 Volume offset:683  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:172 Volume offset:684  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:173 Volume offset:685  Expected:2 Got:4
okShift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:174 Volume offset:686  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:175 Volume offset:687  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:176 Volume offset:688  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:177 Volume offset:689  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:178 Volume offset:690  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:179 Volume offset:691  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:180 Volume offset:692  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:181 Volume offset:693  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:182 Volume offset:694  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:183 Volume offset:695  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:184 Volume offset:696  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:185 Volume offset:697  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:186 Volume offset:698  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:187 Volume offset:699  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:188 Volume offset:700  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:189 Volume offset:701  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:190 Volume offset:702  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:191 Volume offset:703  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:192 Volume offset:704  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:193 Volume offset:705  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:194 Volume offset:706  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:195 Volume offset:707  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:196 Volume offset:708  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:197 Volume offset:709  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:198 Volume offset:710  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:199 Volume offset:711  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:200 Volume offset:712  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:201 Volume offset:713  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:202 Volume offset:714  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:203 Volume offset:715  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:204 Volume offset:716  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:205 Volume offset:717  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:206 Volume offset:718  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:207 Volume offset:719  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:208 Volume offset:720  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2

Out of Range Block::1 bo:209 Volume offset:721  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:210 Volume offset:722  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:211 Volume offset:723  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:212 Volume offset:724  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:213 Volume offset:725  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:214 Volume offset:726  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:215 Volume offset:727  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:216 Volume offset:728  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:217 Volume offset:729  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:218 Volume offset:730  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:219 Volume offset:731  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:220 Volume offset:732  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:221 Volume offset:733  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:222 Volume offset:734  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:223 Volume offset:735  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:224 Volume offset:736  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:225 Volume offset:737  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:226 Volume offset:738  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:227 Volume offset:739  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:228 Volume offset:740  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:229 Volume offset:741  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:230 Volume offset:742  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:231 Volume offset:743  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:232 Volume offset:744  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:233 Volume offset:745  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:234 Volume offset:746  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:235 Volume offset:747  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:236 Volume offset:748  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:237 Volume offset:749  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:238 Volume offset:750  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:239 Volume offset:751  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:240 Volume offset:752  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:241 Volume offset:753  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:242 Volume offset:754  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:243 Volume offset:755  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:244 Volume offset:756  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:245 Volume offset:757  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:246 Volume offset:758  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:247 Volume offset:759  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:248 Volume offset:760  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:249 Volume offset:761  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:250 Volume offset:762  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:251 Volume offset:763  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:252 Volume offset:764  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:253 Volume offset:765  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:254 Volume offset:766  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:255 Volume offset:767  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:256 Volume offset:768  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:257 Volume offset:769  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:258 Volume offset:770  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:259 Volume offset:771  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
test test::test_wl_update_rollover ... Out of Range Block::1 bo:260 Volume offset:772  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:261 Volume offset:773  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:262 Volume offset:774  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:263 Volume offset:775  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:264 Volume offset:776  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:265 Volume offset:777  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:266 Volume offset:778  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:267 Volume offset:779  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:268 Volume offset:780  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:269 Volume offset:781  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:270 Volume offset:782  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:271 Volume offset:783  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:272 Volume offset:784  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:273 Volume offset:785  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:274 Volume offset:786  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:275 Volume offset:787  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:276 Volume offset:788  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:277 Volume offset:789  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:278 Volume offset:790  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:279 Volume offset:791  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:280 Volume offset:792  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:281 Volume offset:793  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:282 Volume offset:794  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:283 Volume offset:795  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:284 Volume offset:796  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:285 Volume offset:797  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:286 Volume offset:798  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:287 Volume offset:799  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:288 Volume offset:800  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:289 Volume offset:801  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:290 Volume offset:802  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:291 Volume offset:803  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:292 Volume offset:804  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:293 Volume offset:805  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:294 Volume offset:806  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:295 Volume offset:807  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:296 Volume offset:808  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:297 Volume offset:809  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:298 Volume offset:810  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:299 Volume offset:811  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:300 Volume offset:812  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:301 Volume offset:813  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:302 Volume offset:814  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:303 Volume offset:815  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:304 Volume offset:816  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:305 Volume offset:817  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:306 Volume offset:818  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:307 Volume offset:819  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:308 Volume offset:820  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:309 Volume offset:821  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:310 Volume offset:822  Expected:2 Got:4
okShift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:311 Volume offset:823  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:312 Volume offset:824  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:313 Volume offset:825  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:314 Volume offset:826  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:315 Volume offset:827  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:316 Volume offset:828  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:317 Volume offset:829  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:318 Volume offset:830  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:319 Volume offset:831  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:320 Volume offset:832  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:321 Volume offset:833  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:322 Volume offset:834  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:323 Volume offset:835  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:324 Volume offset:836  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:325 Volume offset:837  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:326 Volume offset:838  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:327 Volume offset:839  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:328 Volume offset:840  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:329 Volume offset:841  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:330 Volume offset:842  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:331 Volume offset:843  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:332 Volume offset:844  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:333 Volume offset:845  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:334 Volume offset:846  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:335 Volume offset:847  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:336 Volume offset:848  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:337 Volume offset:849  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:338 Volume offset:850  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:339 Volume offset:851  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:340 Volume offset:852  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:341 Volume offset:853  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2

Out of Range Block::1 bo:342 Volume offset:854  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:343 Volume offset:855  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:344 Volume offset:856  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:345 Volume offset:857  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:346 Volume offset:858  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:347 Volume offset:859  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:348 Volume offset:860  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:349 Volume offset:861  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:350 Volume offset:862  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:351 Volume offset:863  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:352 Volume offset:864  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:353 Volume offset:865  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:354 Volume offset:866  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:355 Volume offset:867  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:356 Volume offset:868  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:357 Volume offset:869  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:358 Volume offset:870  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:359 Volume offset:871  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:360 Volume offset:872  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:361 Volume offset:873  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:362 Volume offset:874  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:363 Volume offset:875  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:364 Volume offset:876  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:365 Volume offset:877  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:366 Volume offset:878  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:367 Volume offset:879  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:368 Volume offset:880  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:369 Volume offset:881  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:370 Volume offset:882  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:371 Volume offset:883  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:372 Volume offset:884  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:373 Volume offset:885  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:374 Volume offset:886  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:375 Volume offset:887  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:376 Volume offset:888  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:377 Volume offset:889  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:378 Volume offset:890  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:379 Volume offset:891  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:380 Volume offset:892  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:381 Volume offset:893  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:382 Volume offset:894  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:383 Volume offset:895  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:384 Volume offset:896  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:385 Volume offset:897  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:386 Volume offset:898  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:387 Volume offset:899  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:388 Volume offset:900  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:389 Volume offset:901  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:390 Volume offset:902  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:391 Volume offset:903  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:392 Volume offset:904  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:393 Volume offset:905  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:394 Volume offset:906  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:395 Volume offset:907  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:396 Volume offset:908  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:397 Volume offset:909  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:398 Volume offset:910  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:399 Volume offset:911  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:400 Volume offset:912  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:401 Volume offset:913  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:402 Volume offset:914  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:403 Volume offset:915  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:404 Volume offset:916  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:405 Volume offset:917  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:406 Volume offset:918  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:407 Volume offset:919  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:408 Volume offset:920  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:409 Volume offset:921  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:410 Volume offset:922  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:411 Volume offset:923  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:412 Volume offset:924  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:413 Volume offset:925  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:414 Volume offset:926  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:415 Volume offset:927  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:416 Volume offset:928  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:417 Volume offset:929  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:418 Volume offset:930  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:419 Volume offset:931  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:420 Volume offset:932  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:421 Volume offset:933  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:422 Volume offset:934  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:423 Volume offset:935  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:424 Volume offset:936  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:425 Volume offset:937  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:426 Volume offset:938  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:427 Volume offset:939  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:428 Volume offset:940  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:429 Volume offset:941  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:430 Volume offset:942  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:431 Volume offset:943  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:432 Volume offset:944  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:433 Volume offset:945  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:434 Volume offset:946  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:435 Volume offset:947  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:436 Volume offset:948  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:437 Volume offset:949  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:438 Volume offset:950  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:439 Volume offset:951  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:440 Volume offset:952  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:441 Volume offset:953  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:442 Volume offset:954  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:443 Volume offset:955  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:444 Volume offset:956  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:445 Volume offset:957  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:446 Volume offset:958  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:447 Volume offset:959  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:448 Volume offset:960  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:449 Volume offset:961  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:450 Volume offset:962  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:451 Volume offset:963  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:452 Volume offset:964  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:453 Volume offset:965  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:454 Volume offset:966  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:455 Volume offset:967  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:456 Volume offset:968  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:457 Volume offset:969  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:458 Volume offset:970  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:459 Volume offset:971  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:460 Volume offset:972  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:461 Volume offset:973  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:462 Volume offset:974  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:463 Volume offset:975  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:464 Volume offset:976  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:465 Volume offset:977  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:466 Volume offset:978  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:467 Volume offset:979  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:468 Volume offset:980  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:469 Volume offset:981  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:470 Volume offset:982  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:471 Volume offset:983  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:472 Volume offset:984  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:473 Volume offset:985  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:474 Volume offset:986  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:475 Volume offset:987  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:476 Volume offset:988  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:477 Volume offset:989  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:478 Volume offset:990  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:479 Volume offset:991  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:480 Volume offset:992  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:481 Volume offset:993  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:482 Volume offset:994  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:483 Volume offset:995  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:484 Volume offset:996  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:485 Volume offset:997  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:486 Volume offset:998  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:487 Volume offset:999  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:488 Volume offset:1000  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:489 Volume offset:1001  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:490 Volume offset:1002  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:491 Volume offset:1003  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:492 Volume offset:1004  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:493 Volume offset:1005  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:494 Volume offset:1006  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:495 Volume offset:1007  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:496 Volume offset:1008  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:497 Volume offset:1009  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:498 Volume offset:1010  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:499 Volume offset:1011  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:500 Volume offset:1012  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:501 Volume offset:1013  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:502 Volume offset:1014  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:503 Volume offset:1015  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:504 Volume offset:1016  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:505 Volume offset:1017  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:506 Volume offset:1018  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:507 Volume offset:1019  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:508 Volume offset:1020  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:509 Volume offset:1021  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:510 Volume offset:1022  Expected:2 Got:4
Shift 0, v:4 sv:4 min:2 cur:2
Out of Range Block::1 bo:511 Volume offset:1023  Expected:2 Got:4
test test::test_wl_commit_range_vv ... ok

test result: ok. 64 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s

     Running `/work/oxidecomputer/crucible/target/debug/deps/dsc-2b80980cbc3bac2c --nocapture`

running 16 tests
Creating region directory at: /tmp/.tmpFyEcKy
Creating region directory at: /tmp/.tmpneCf4N
Creating region directory at: /tmp/.tmpQfbQmF
Creating dsc directory at: /tmp/.tmpa1Clwx
Creating region directory at: /tmp/.tmpb9iNSL
Creating region directory at: /tmp/.tmphjd71E
Creating region directory at: /tmp/.tmpU83AJS
Creating dsc directory at: /tmp/.tmpWQECQf
Creating region directory at: /tmp/.tmp8fIrtP
Creating region directory at: /tmp/.tmptkedSS
Creating region directory at: /tmp/.tmpTiSUvs
Creating region directory at: /tmp/.tmp9bCgVj
Creating region directory at: /tmp/.tmpEJudHw
Creating dsc directory at: /tmp/.tmpBz9a5a
res is Err(No such file or directory (os error 2))
test test::bad_bin ... ok
test test::new_ti_four ... ok
test test::new_ti ... ok
test test::existing_ti ... ok
test test::delete_bad_second_region ... ok
test test::delete_region ... ok
test test::delete_bad_region ... ok
Creating region directory at: /tmp/.tmpXXKa1h
test test::port_to_region_generation ... ok
Creating region directory at: /tmp/.tmpSUuWSI
Creating region directory at: /tmp/.tmp8hG3wS
Creating dsc directory at: /tmp/.tmp4DPP60
test test::new_ti_two_dirs ... ok
Creating region directory at: /tmp/.tmpRlI2MK
test test::new_ti_three ... ok
Creating region directory at: /tmp/.tmpUr9WQJ
Creating region directory at: /tmp/.tmpwqOD5v
Creating region directory at: /tmp/.tmpILYztV
test test::new_ti_two_region_count ... ok
test test::restart_four_region ... ok
test test::restart_three_region ... ok
test test::restart_region_bad ... ok
test test::restart_region_four_bad ... ok
test control::test::test_dsc_openapi ... ok

test result: ok. 16 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s

     Running `/work/oxidecomputer/crucible/target/debug/deps/dsc_client-f8a5b497695371e1 --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/measure_iops-cbdca99bf515defe --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

     Running `/work/oxidecomputer/crucible/target/debug/deps/repair_client-5353c8de97b4615f --nocapture`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible --test /work/oxidecomputer/crucible/upstairs/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern aes_gcm_siv=/work/oxidecomputer/crucible/target/debug/deps/libaes_gcm_siv-21495b616a07c9a4.rlib --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern async_recursion=/work/oxidecomputer/crucible/target/debug/deps/libasync_recursion-ce9499495a1cb858.so --extern async_trait=/work/oxidecomputer/crucible/target/debug/deps/libasync_trait-a300e84178ee0ad1.so --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern itertools=/work/oxidecomputer/crucible/target/debug/deps/libitertools-b06e69badd72e55c.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern proptest=/work/oxidecomputer/crucible/target/debug/deps/libproptest-327f7f2cf6858f27.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern test_strategy=/work/oxidecomputer/crucible/target/debug/deps/libtest_strategy-5eb6b90d55d9f739.so --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern tokio_test=/work/oxidecomputer/crucible/target/debug/deps/libtokio_test-12a28be646ff63e6.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rlib --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-agent-client
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_agent_client --test /work/oxidecomputer/crucible/agent-client/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern crucible_agent_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_agent_client-86e1c18945d61be3.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-client-types
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_client_types --test /work/oxidecomputer/crucible/crucible-client-types/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-common
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_common --test /work/oxidecomputer/crucible/common/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern atty=/work/oxidecomputer/crucible/target/debug/deps/libatty-bfb6a2cdc762f7c4.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rlib --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rlib --extern rustls_pemfile=/work/oxidecomputer/crucible/target/debug/deps/librustls_pemfile-e52b2a6debfcae48.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_bunyan=/work/oxidecomputer/crucible/target/debug/deps/libslog_bunyan-dce051a6775f1d99.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern twox_hash=/work/oxidecomputer/crucible/target/debug/deps/libtwox_hash-9f5dd4f7319ca539.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-control-client
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_control_client --test /work/oxidecomputer/crucible/control-client/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern crucible_control_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_control_client-3d0142c7d3790e17.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-downstairs
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_downstairs --test /work/oxidecomputer/crucible/downstairs/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_downstairs=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_downstairs-6276be71be5284a4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern hyper_staticfile=/work/oxidecomputer/crucible/target/debug/deps/libhyper_staticfile-559b4389ef952563.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern mime_guess=/work/oxidecomputer/crucible/target/debug/deps/libmime_guess-66974d6c31968dc2.rlib --extern nix=/work/oxidecomputer/crucible/target/debug/deps/libnix-3635aff9412bf811.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry-5524fd7817ad57fb.rlib --extern opentelemetry_jaeger=/work/oxidecomputer/crucible/target/debug/deps/libopentelemetry_jaeger-9ebf47742e5e063f.rlib --extern oximeter=/work/oxidecomputer/crucible/target/debug/deps/liboximeter-90ae047d6b643e4e.rlib --extern oximeter_producer=/work/oxidecomputer/crucible/target/debug/deps/liboximeter_producer-5dc4f732e258486e.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern rand_chacha=/work/oxidecomputer/crucible/target/debug/deps/librand_chacha-add466c063ef8725.rlib --extern repair_client=/work/oxidecomputer/crucible/target/debug/deps/librepair_client-1452d56087b6ccb7.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern ringbuffer=/work/oxidecomputer/crucible/target/debug/deps/libringbuffer-e91d75e4694e6351.rlib --extern rusqlite=/work/oxidecomputer/crucible/target/debug/deps/librusqlite-e4d2316a88b06837.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_rustls=/work/oxidecomputer/crucible/target/debug/deps/libtokio_rustls-eafe4ab74a176b7d.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern toml=/work/oxidecomputer/crucible/target/debug/deps/libtoml-de0eb3fcc3b95b5c.rlib --extern tracing=/work/oxidecomputer/crucible/target/debug/deps/libtracing-1de351a7f2b0ab55.rlib --extern tracing_opentelemetry=/work/oxidecomputer/crucible/target/debug/deps/libtracing_opentelemetry-8f8cfc1900c3a663.rlib --extern tracing_subscriber=/work/oxidecomputer/crucible/target/debug/deps/libtracing_subscriber-e39dae5ba339bc78.rlib --extern usdt=/work/oxidecomputer/crucible/target/debug/deps/libusdt-86bb76e3b8fcea87.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --cfg 'feature="default"' --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-integration-tests
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_integration_tests --test /work/oxidecomputer/crucible/integration_tests/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_client_types=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_client_types-bd54c4335d2370bd.rlib --extern crucible_downstairs=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_downstairs-6276be71be5284a4.rlib --extern crucible_integration_tests=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_integration_tests-521d4724b4b30c4a.rlib --extern crucible_pantry=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry-fe633af5059fe3a7.rlib --extern crucible_pantry_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry_client-ccb9ddeebb23cea2.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern futures_core=/work/oxidecomputer/crucible/target/debug/deps/libfutures_core-46c6e3a1b3966417.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern httptest=/work/oxidecomputer/crucible/target/debug/deps/libhttptest-174da737d96e2af6.rlib --extern rand=/work/oxidecomputer/crucible/target/debug/deps/librand-1f91a9ea4aed49ee.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern slog_async=/work/oxidecomputer/crucible/target/debug/deps/libslog_async-03e743f036119074.rlib --extern slog_dtrace=/work/oxidecomputer/crucible/target/debug/deps/libslog_dtrace-8024beacfb95325b.rlib --extern slog_term=/work/oxidecomputer/crucible/target/debug/deps/libslog_term-31c687431f6dd53c.rlib --extern tempfile=/work/oxidecomputer/crucible/target/debug/deps/libtempfile-05cb6a5f7e86cb66.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-pantry
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_pantry --test /work/oxidecomputer/crucible/pantry/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern base64=/work/oxidecomputer/crucible/target/debug/deps/libbase64-c139bdd129e780ac.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible=/work/oxidecomputer/crucible/target/debug/deps/libcrucible-41ca439abdc23695.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_pantry=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry-fe633af5059fe3a7.rlib --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern dropshot=/work/oxidecomputer/crucible/target/debug/deps/libdropshot-a49a4505c9c6b86f.rlib --extern expectorate=/work/oxidecomputer/crucible/target/debug/deps/libexpectorate-4e45b262baa473cc.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern hex=/work/oxidecomputer/crucible/target/debug/deps/libhex-6531b11cb72de3e5.rlib --extern http=/work/oxidecomputer/crucible/target/debug/deps/libhttp-4526ab951518c907.rlib --extern hyper=/work/oxidecomputer/crucible/target/debug/deps/libhyper-dad943d3b7cc33e9.rlib --extern omicron_common=/work/oxidecomputer/crucible/target/debug/deps/libomicron_common-52df1fff8b36d94c.rlib --extern openapi_lint=/work/oxidecomputer/crucible/target/debug/deps/libopenapi_lint-848bb99097a9a843.rlib --extern openapiv3=/work/oxidecomputer/crucible/target/debug/deps/libopenapiv3-f84072cef6d0d68c.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern sha2=/work/oxidecomputer/crucible/target/debug/deps/libsha2-9b09b4b286e2cb62.rlib --extern slog=/work/oxidecomputer/crucible/target/debug/deps/libslog-84fd25666c3c26ee.rlib --extern subprocess=/work/oxidecomputer/crucible/target/debug/deps/libsubprocess-0acfc5c9b903588a.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-pantry-client
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_pantry_client --test /work/oxidecomputer/crucible/pantry-client/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern crucible_pantry_client=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_pantry_client-ccb9ddeebb23cea2.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-protocol
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_protocol --test /work/oxidecomputer/crucible/protocol/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern bincode=/work/oxidecomputer/crucible/target/debug/deps/libbincode-bcb925e8faac86cd.rlib --extern bytes=/work/oxidecomputer/crucible/target/debug/deps/libbytes-64a8a55ef81e55dd.rlib --extern clap=/work/oxidecomputer/crucible/target/debug/deps/libclap-49bc17aade028e79.rlib --extern crucible_common=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_common-c45901e9152d33f4.rlib --extern crucible_protocol=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_protocol-0682e169c907a102.rlib --extern futures=/work/oxidecomputer/crucible/target/debug/deps/libfutures-df761c89bfa71e54.rlib --extern num_enum=/work/oxidecomputer/crucible/target/debug/deps/libnum_enum-9cd7a6d9dcf1dd5a.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern statistical=/work/oxidecomputer/crucible/target/debug/deps/libstatistical-f2809857d7fe1c91.rlib --extern tokio=/work/oxidecomputer/crucible/target/debug/deps/libtokio-cf190744403b2ee1.rlib --extern tokio_util=/work/oxidecomputer/crucible/target/debug/deps/libtokio_util-279b3765a2b5aad1.rlib --extern uuid=/work/oxidecomputer/crucible/target/debug/deps/libuuid-7cc8e87b4149b49e.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests crucible-smf
     Running `rustdoc --edition=2021 --crate-type lib --crate-name crucible_smf --test /work/oxidecomputer/crucible/smf/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern crucible_smf=/work/oxidecomputer/crucible/target/debug/deps/libcrucible_smf-e1a82c6f17385dc6.rlib --extern libc=/work/oxidecomputer/crucible/target/debug/deps/liblibc-a748caf4ceff51bd.rlib --extern num_derive=/work/oxidecomputer/crucible/target/debug/deps/libnum_derive-814c8a0a0a713cba.so --extern num_traits=/work/oxidecomputer/crucible/target/debug/deps/libnum_traits-8e50de91aba3f8f9.rlib --extern thiserror=/work/oxidecomputer/crucible/target/debug/deps/libthiserror-07eca56f531a0e5d.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests dsc-client
     Running `rustdoc --edition=2021 --crate-type lib --crate-name dsc_client --test /work/oxidecomputer/crucible/dsc-client/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern dsc_client=/work/oxidecomputer/crucible/target/debug/deps/libdsc_client-15b0c81fa833cf0f.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s

   Doc-tests repair-client
     Running `rustdoc --edition=2021 --crate-type lib --crate-name repair_client --test /work/oxidecomputer/crucible/repair-client/src/lib.rs -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L dependency=/work/oxidecomputer/crucible/target/debug/deps -L native=/usr/lib/amd64 -L native=/usr/ssl-3/lib/amd64 -L native=/work/oxidecomputer/crucible/target/debug/build/libgit2-sys-6120c1544d9c7fd8/out/build -L native=/work/oxidecomputer/crucible/target/debug/build/ring-2b909d7f60e3ea12/out --test-args --nocapture --extern anyhow=/work/oxidecomputer/crucible/target/debug/deps/libanyhow-c23d7ea2a714731e.rlib --extern chrono=/work/oxidecomputer/crucible/target/debug/deps/libchrono-0402a5e17dd99cdf.rlib --extern percent_encoding=/work/oxidecomputer/crucible/target/debug/deps/libpercent_encoding-0000aebce3d30803.rlib --extern progenitor=/work/oxidecomputer/crucible/target/debug/deps/libprogenitor-0319e0dfd841f493.rlib --extern repair_client=/work/oxidecomputer/crucible/target/debug/deps/librepair_client-1452d56087b6ccb7.rlib --extern reqwest=/work/oxidecomputer/crucible/target/debug/deps/libreqwest-6407fc4e9374ca8e.rlib --extern schemars=/work/oxidecomputer/crucible/target/debug/deps/libschemars-83d20014cee5d9b5.rlib --extern serde=/work/oxidecomputer/crucible/target/debug/deps/libserde-2779165e31567af2.rlib --extern serde_json=/work/oxidecomputer/crucible/target/debug/deps/libserde_json-96027e7dd982d07a.rlib -C embed-bitcode=no --error-format human`

running 0 tests

test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s


real    16:26.588377952
user    52:35.118836291
sys      5:26.872311264
trap        1.675553487
tflt        0.373093867
dflt        1.700465352
kflt        0.000709715
lock  1:53:33.351688251
slp   2:02:19.411583747
lat      1:25.000689190
stop       50.092823530