01HQRV876FZ3ZMNX15TNJFXM5W: test-ds

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HQRV8KWWD1YPH68Y5TQKF13Y

Tags:

Output:

SEQ GLOBAL TIME DETAILS
12024-02-28T22:09:22.251Zjob dependencies complete; ready to run (waiting for 16 m 59 s)
22024-02-28T22:10:10.928Zjob assigned to worker 01HQRW7T1MN4NVBB7P2C0FKEAS (queued for 48 s)
32024-02-28T22:10:11.896Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42024-02-28T22:10:37.727Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52024-02-28T22:10:37.727Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62024-02-28T22:10:59.767Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72024-02-28T22:10:59.767Zdownloading input: /input/build/work/bins/crutest.gz
82024-02-28T22:11:28.343Zdownloaded input: /input/build/work/bins/crutest.gz
92024-02-28T22:11:28.343Zdownloading input: /input/build/work/bins/dsc.gz
102024-02-28T22:11:44.473Zdownloaded input: /input/build/work/bins/dsc.gz
112024-02-28T22:11:44.485Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122024-02-28T22:11:44.523Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132024-02-28T22:11:44.524Zdownloading input: /input/build/work/scripts/test_ds.sh
142024-02-28T22:11:44.573Zdownloaded input: /input/build/work/scripts/test_ds.sh
152024-02-28T22:11:44.573Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162024-02-28T22:11:44.612Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172024-02-28T22:11:44.612Zdownloading input: /input/build/work/scripts/test_repair.sh
182024-02-28T22:11:44.642Zdownloaded input: /input/build/work/scripts/test_repair.sh
192024-02-28T22:11:44.642Zdownloading input: /input/build/work/scripts/test_replay.sh
202024-02-28T22:11:44.681Zdownloaded input: /input/build/work/scripts/test_replay.sh
212024-02-28T22:11:44.681Zdownloading input: /input/build/work/scripts/test_up.sh
222024-02-28T22:11:44.716Zdownloaded input: /input/build/work/scripts/test_up.sh
232024-02-28T22:11:44.716Zdownloading input: /input/build/work/scripts/upstairs_info.d
242024-02-28T22:11:44.736Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252024-02-28T22:11:44.744Zdownloading input: /input/build/tmp/cargo-test-out.log
262024-02-28T22:12:32.323Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272024-02-28T22:12:32.323Zstarting task 0: "setup"
282024-02-28T22:12:32.690Z++ uname -s
292024-02-28T22:12:32.730Z+ kern=SunOS
302024-02-28T22:12:32.730Z+ case "$kern" in
312024-02-28T22:12:32.730Z+ groupadd -g 12345 build
322024-02-28T22:12:32.730Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332024-02-28T22:12:34.710Z+ zfs create -o mountpoint=/work rpool/work
342024-02-28T22:12:34.827Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352024-02-28T22:12:34.845Z+ home_fs=zfs
362024-02-28T22:12:34.845Z+ [[ zfs == autofs ]]
372024-02-28T22:12:34.845Z+ mkdir -p /home/build
382024-02-28T22:12:34.845Z+ chown build:build /home/build /work
392024-02-28T22:12:36.828Z+ chmod 0700 /home/build /work
402024-02-28T22:12:36.835Zprocess exited: duration 4499 ms, exit code 0
 
412024-02-28T22:12:36.852Zstarting task 1: "authentication"
422024-02-28T22:12:37.320Zprocess exited: duration 31 ms, exit code 0
 
432024-02-28T22:12:37.353Zstarting task 2: "build"
442024-02-28T22:12:37.799Z+ banner cores
452024-02-28T22:12:37.799Z
462024-02-28T22:12:37.800Z #### #### ##### ###### ####
472024-02-28T22:12:37.800Z # # # # # # # #
482024-02-28T22:12:37.800Z # # # # # ##### ####
492024-02-28T22:12:37.800Z # # # ##### # #
502024-02-28T22:12:37.800Z # # # # # # # # #
512024-02-28T22:12:37.801Z #### #### # # ###### ####
522024-02-28T22:12:37.801Z
532024-02-28T22:12:37.801Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542024-02-28T22:12:37.801Z+ echo 'input bins dir contains:'
552024-02-28T22:12:37.801Zinput bins dir contains:
562024-02-28T22:12:37.801Z+ ls -ltr /input/build/work/bins
572024-02-28T22:12:37.801Ztotal 740779
582024-02-28T22:12:37.801Z-rw-r--r-- 1 root root 109740032 Feb 28 22:10 crucible-downstairs.gz
592024-02-28T22:12:37.801Z-rw-r--r-- 1 root root 91529491 Feb 28 22:10 crucible-hammer.gz
602024-02-28T22:12:37.802Z-rw-r--r-- 1 root root 105639604 Feb 28 22:11 crutest.gz
612024-02-28T22:12:37.802Z-rw-r--r-- 1 root root 72121864 Feb 28 22:11 dsc.gz
622024-02-28T22:12:37.802Z+ banner unpack
632024-02-28T22:12:37.802Z
642024-02-28T22:12:37.802Z # # # # ##### ## #### # #
652024-02-28T22:12:37.802Z # # ## # # # # # # # # #
662024-02-28T22:12:37.802Z # # # # # # # # # # ####
672024-02-28T22:12:37.802Z # # # # # ##### ###### # # #
682024-02-28T22:12:37.802Z # # # ## # # # # # # #
692024-02-28T22:12:37.802Z #### # # # # # #### # #
702024-02-28T22:12:37.802Z
712024-02-28T22:12:37.803Z+ mkdir -p /var/tmp/bins
722024-02-28T22:12:37.803Z+ for t in "$input/bins/"*.gz
732024-02-28T22:12:37.803Z++ basename /input/build/work/bins/crucible-downstairs.gz
742024-02-28T22:12:37.803Z+ b=crucible-downstairs.gz
752024-02-28T22:12:37.803Z+ b=crucible-downstairs
762024-02-28T22:12:37.803Z+ gunzip
772024-02-28T22:12:40.078Z+ chmod +x /var/tmp/bins/crucible-downstairs
782024-02-28T22:12:40.081Z+ for t in "$input/bins/"*.gz
792024-02-28T22:12:40.082Z++ basename /input/build/work/bins/crucible-hammer.gz
802024-02-28T22:12:40.088Z+ b=crucible-hammer.gz
812024-02-28T22:12:40.088Z+ b=crucible-hammer
822024-02-28T22:12:40.088Z+ gunzip
832024-02-28T22:12:42.404Z+ chmod +x /var/tmp/bins/crucible-hammer
842024-02-28T22:12:42.404Z+ for t in "$input/bins/"*.gz
852024-02-28T22:12:42.404Z++ basename /input/build/work/bins/crutest.gz
862024-02-28T22:12:42.404Z+ b=crutest.gz
872024-02-28T22:12:42.404Z+ b=crutest
882024-02-28T22:12:42.404Z+ gunzip
892024-02-28T22:12:44.719Z+ chmod +x /var/tmp/bins/crutest
902024-02-28T22:12:44.755Z+ for t in "$input/bins/"*.gz
912024-02-28T22:12:44.755Z++ basename /input/build/work/bins/dsc.gz
922024-02-28T22:12:44.755Z+ b=dsc.gz
932024-02-28T22:12:44.755Z+ b=dsc
942024-02-28T22:12:44.755Z+ gunzip
952024-02-28T22:12:46.429Z+ chmod +x /var/tmp/bins/dsc
962024-02-28T22:12:46.452Z+ export BINDIR=/var/tmp/bins
972024-02-28T22:12:46.452Z+ BINDIR=/var/tmp/bins
982024-02-28T22:12:46.452Z+ banner test_ds
992024-02-28T22:12:46.452Z
1002024-02-28T22:12:46.452Z ##### ###### #### ##### ##### ####
1012024-02-28T22:12:46.452Z # # # # # # #
1022024-02-28T22:12:46.452Z # ##### #### # # # ####
1032024-02-28T22:12:46.452Z # # # # # # #
1042024-02-28T22:12:46.453Z # # # # # # # # #
1052024-02-28T22:12:46.453Z # ###### #### # ####### ##### ####
1062024-02-28T22:12:46.453Z
1072024-02-28T22:12:46.453Z+ ptime -m bash /input/build/work/scripts/test_ds.sh
1082024-02-28T22:12:46.539Z/input/build/work
1092024-02-28T22:12:46.547ZCreate file for import
1102024-02-28T22:12:46.619Z300+0 records in
1112024-02-28T22:12:46.620Z300+0 records out
1122024-02-28T22:12:46.620Z153600 bytes (150 KiB) transferred in 0.002430 secs (60 MiB/sec)
1132024-02-28T22:12:46.631ZImport region
1142024-02-28T22:12:46.662Z{"msg":"Created new region file \"/tmp/ds_test/region/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:45.801498011Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1131}
1152024-02-28T22:12:47.087ZImport file_size: 153600 Extent size: 51200 Needed extents: 3
1162024-02-28T22:12:47.087ZRegion already large enough for image
1172024-02-28T22:12:47.087ZImporting "/tmp/ds_test/import" to region
1182024-02-28T22:12:47.101ZPopulated 3 extents by copying 153600 bytes (300 blocks)
1192024-02-28T22:12:47.101Z{"msg":"UUID: 12345678-1234-1234-1234-000000000001","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:45.826074019Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1131}
1202024-02-28T22:12:47.101Z{"msg":"Blocks per extent:100 Total Extents: 15","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:45.826152478Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1131}
1212024-02-28T22:12:47.101ZExport region
1222024-02-28T22:12:47.101Z{"msg":"Opened existing region file \"/tmp/ds_test/region/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:45.846272431Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1132}
1232024-02-28T22:12:47.101Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:45.846526074Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1132}
1242024-02-28T22:12:47.101Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:45.846546781Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1132}
1252024-02-28T22:12:47.101ZExport total_size: 768000 Extent size:51200 Total Extents:15
1262024-02-28T22:12:47.101ZExporting from start_block: 0 count:300
1272024-02-28T22:12:47.102ZRead and wrote out 300 blocks
1282024-02-28T22:12:47.102ZImport Export test passed
1292024-02-28T22:12:47.102ZTest clone
1302024-02-28T22:12:47.102ZStarting downstairs
1312024-02-28T22:12:47.782Z PID TTY TIME CMD
1322024-02-28T22:12:47.782Z 1134 ? 0:00 crucible
1332024-02-28T22:12:47.782ZDownstairs running
1342024-02-28T22:12:47.782ZCreating new downstairs
1352024-02-28T22:12:47.903Z{"msg":"Created new region file \"/tmp/ds_test/clone/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.048860861Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1138}
1362024-02-28T22:12:47.915Z{"msg":"UUID: 915871e3-b07c-4265-8218-b07ef20902d1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.049517904Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1138}
1372024-02-28T22:12:47.915Z{"msg":"Blocks per extent:100 Total Extents: 15","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.04954805Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1138}
1382024-02-28T22:12:47.915ZCloning existing downstairs
1392024-02-28T22:12:47.918Z{"msg":"Opened existing region file \"/tmp/ds_test/clone/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.062215238Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1402024-02-28T22:12:47.918Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.062444855Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1412024-02-28T22:12:47.919Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.062461812Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1422024-02-28T22:12:47.919Z{"msg":"UUID: 915871e3-b07c-4265-8218-b07ef20902d1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.062714645Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1432024-02-28T22:12:47.919Z{"msg":"Blocks per extent:100 Total Extents: 15","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.062740671Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1442024-02-28T22:12:47.919Z{"msg":"Crucible Version: Crucible Version: 0.0.1\nCommit SHA: 80b6f734175c2bc8e222c8d6f7875e53cbd5c9ca\nCommit timestamp: 2024-02-28T21:51:33.000000000Z branch: main\nrustc: 1.75.0 stable x86_64-unknown-illumos\nCargo: x86_64-unknown-illumos Debug: true Opt level: 0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.062830858Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1452024-02-28T22:12:47.919Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.062866173Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1462024-02-28T22:12:47.919Z{"msg":"Connecting to 127.0.0.1:12810 to obtain our extent files.","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.062880601Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1472024-02-28T22:12:47.962Z{"msg":"The source RegionDefinition is: RegionDefinition { block_size: 512, extent_size: Block { value: 100, shift: 9 }, extent_count: 15, uuid: 12345678-1234-1234-1234-000000000001, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.113230132Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1482024-02-28T22:12:47.971Z{"msg":"The source mode is: true","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.113776672Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1492024-02-28T22:12:47.971Z{"msg":"my def is RegionDefinition { block_size: 512, extent_size: Block { value: 100, shift: 9 }, extent_count: 15, uuid: 915871e3-b07c-4265-8218-b07ef20902d1, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.113814456Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1502024-02-28T22:12:47.971Z{"msg":"Repair extent 0","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.113980482Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1512024-02-28T22:12:47.971Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/000.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.114016357Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1522024-02-28T22:12:48.015Z{"msg":"eid:0 Found repair files: [\"000\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.147234656Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1532024-02-28T22:12:48.035Z{"msg":"Verify extent 0 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.149209245Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1542024-02-28T22:12:48.035Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/000.copy\" to \"/tmp/ds_test/clone/00/000/000.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.14979141Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1552024-02-28T22:12:48.035Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/000.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.150445364Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1562024-02-28T22:12:48.036Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/000.replace\" to \"/tmp/ds_test/clone/00/000/000.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.150563216Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1572024-02-28T22:12:48.036Z{"msg":"Repair extent 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.150644175Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1582024-02-28T22:12:48.036Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/001.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.15067257Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1592024-02-28T22:12:48.045Z{"msg":"eid:1 Found repair files: [\"001\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.18143666Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1602024-02-28T22:12:48.045Z{"msg":"Verify extent 1 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.183040624Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1612024-02-28T22:12:48.046Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/001.copy\" to \"/tmp/ds_test/clone/00/000/001.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.183606891Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1622024-02-28T22:12:48.046Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/001.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.184058925Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1632024-02-28T22:12:48.046Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/001.replace\" to \"/tmp/ds_test/clone/00/000/001.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.184111407Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1642024-02-28T22:12:48.046Z{"msg":"Repair extent 2","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.184129334Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1652024-02-28T22:12:48.047Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/002.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.184151851Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1662024-02-28T22:12:48.072Z{"msg":"eid:2 Found repair files: [\"002\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.214313399Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1672024-02-28T22:12:48.093Z{"msg":"Verify extent 2 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.215778624Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1682024-02-28T22:12:48.093Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/002.copy\" to \"/tmp/ds_test/clone/00/000/002.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.216318064Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1692024-02-28T22:12:48.093Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/002.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.216742942Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1702024-02-28T22:12:48.094Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/002.replace\" to \"/tmp/ds_test/clone/00/000/002.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.21689482Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1712024-02-28T22:12:48.094Z{"msg":"Repair extent 3","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.216919556Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1722024-02-28T22:12:48.094Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/003.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.216934084Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1732024-02-28T22:12:48.110Z{"msg":"eid:3 Found repair files: [\"003\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.247322478Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1742024-02-28T22:12:48.110Z{"msg":"Verify extent 3 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.248828467Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1752024-02-28T22:12:48.110Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/003.copy\" to \"/tmp/ds_test/clone/00/000/003.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.249319835Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1762024-02-28T22:12:48.110Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/003.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.249708998Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1772024-02-28T22:12:48.110Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/003.replace\" to \"/tmp/ds_test/clone/00/000/003.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.249784117Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1782024-02-28T22:12:48.110Z{"msg":"Repair extent 4","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.249800734Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1792024-02-28T22:12:48.111Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/004.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.24983021Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1802024-02-28T22:12:48.134Z{"msg":"eid:4 Found repair files: [\"004\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.28038451Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1812024-02-28T22:12:48.146Z{"msg":"Verify extent 4 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.281771247Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1822024-02-28T22:12:48.146Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/004.copy\" to \"/tmp/ds_test/clone/00/000/004.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.282273982Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1832024-02-28T22:12:48.146Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/004.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.282677843Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1842024-02-28T22:12:48.147Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/004.replace\" to \"/tmp/ds_test/clone/00/000/004.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.282759331Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1852024-02-28T22:12:48.147Z{"msg":"Repair extent 5","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.282805375Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1862024-02-28T22:12:48.147Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/005.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.282844639Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1872024-02-28T22:12:48.164Z{"msg":"eid:5 Found repair files: [\"005\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.31277948Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1882024-02-28T22:12:48.190Z{"msg":"Verify extent 5 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.314357938Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1892024-02-28T22:12:48.190Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/005.copy\" to \"/tmp/ds_test/clone/00/000/005.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.314853495Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1902024-02-28T22:12:48.191Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/005.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.315346863Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1912024-02-28T22:12:48.191Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/005.replace\" to \"/tmp/ds_test/clone/00/000/005.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.315471735Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1922024-02-28T22:12:48.191Z{"msg":"Repair extent 6","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.31550075Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
1932024-02-28T22:12:48.191Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/006.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.315518797Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1942024-02-28T22:12:48.200Z{"msg":"eid:6 Found repair files: [\"006\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.345795608Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1952024-02-28T22:12:48.200Z{"msg":"Verify extent 6 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.347178565Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1962024-02-28T22:12:48.201Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/006.copy\" to \"/tmp/ds_test/clone/00/000/006.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.347674483Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1972024-02-28T22:12:48.201Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/006.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.348065955Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1982024-02-28T22:12:48.201Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/006.replace\" to \"/tmp/ds_test/clone/00/000/006.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.348188627Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
1992024-02-28T22:12:48.201Z{"msg":"Repair extent 7","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.348226581Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
2002024-02-28T22:12:48.201Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/007.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.348242619Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2012024-02-28T22:12:48.227Z{"msg":"eid:7 Found repair files: [\"007\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.378404807Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2022024-02-28T22:12:48.232Z{"msg":"Verify extent 7 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.379897397Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2032024-02-28T22:12:48.232Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/007.copy\" to \"/tmp/ds_test/clone/00/000/007.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.380419301Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2042024-02-28T22:12:48.232Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/007.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.380828931Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2052024-02-28T22:12:48.232Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/007.replace\" to \"/tmp/ds_test/clone/00/000/007.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.380881773Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2062024-02-28T22:12:48.232Z{"msg":"Repair extent 8","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.380906459Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
2072024-02-28T22:12:48.233Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/008.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.380938035Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2082024-02-28T22:12:48.262Z{"msg":"eid:8 Found repair files: [\"008\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.411023794Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2092024-02-28T22:12:48.781Z{"msg":"Verify extent 8 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.412598592Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2102024-02-28T22:12:48.782Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/008.copy\" to \"/tmp/ds_test/clone/00/000/008.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.413146322Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2112024-02-28T22:12:48.782Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/008.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.413554312Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2122024-02-28T22:12:48.782Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/008.replace\" to \"/tmp/ds_test/clone/00/000/008.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.413626351Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2132024-02-28T22:12:48.782Z{"msg":"Repair extent 9","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.413665276Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
2142024-02-28T22:12:48.782Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/009.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.413688602Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2152024-02-28T22:12:48.789Z{"msg":"eid:9 Found repair files: [\"009\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.443756964Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2162024-02-28T22:12:48.790Z{"msg":"Verify extent 9 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.445198542Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2172024-02-28T22:12:48.790Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/009.copy\" to \"/tmp/ds_test/clone/00/000/009.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.445695329Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2182024-02-28T22:12:48.790Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/009.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.44609969Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2192024-02-28T22:12:48.790Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/009.replace\" to \"/tmp/ds_test/clone/00/000/009.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.446187337Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2202024-02-28T22:12:48.790Z{"msg":"Repair extent 10","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.446221552Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
2212024-02-28T22:12:48.790Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/00A.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.446245558Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2222024-02-28T22:12:48.790Z{"msg":"eid:10 Found repair files: [\"00A\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.476422844Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2232024-02-28T22:12:48.790Z{"msg":"Verify extent 10 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.477970877Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2242024-02-28T22:12:48.790Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/00A.copy\" to \"/tmp/ds_test/clone/00/000/00A.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.478469533Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2252024-02-28T22:12:48.790Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/00A.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.478887342Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2262024-02-28T22:12:48.790Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/00A.replace\" to \"/tmp/ds_test/clone/00/000/00A.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.478985907Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2272024-02-28T22:12:48.790Z{"msg":"Repair extent 11","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.479019412Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
2282024-02-28T22:12:48.790Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/00B.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.479040419Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2292024-02-28T22:12:48.790Z{"msg":"eid:11 Found repair files: [\"00B\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.509585461Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2302024-02-28T22:12:48.790Z{"msg":"Verify extent 11 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.511001893Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2312024-02-28T22:12:48.790Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/00B.copy\" to \"/tmp/ds_test/clone/00/000/00B.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.511489611Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2322024-02-28T22:12:48.790Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/00B.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.511913079Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2332024-02-28T22:12:48.790Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/00B.replace\" to \"/tmp/ds_test/clone/00/000/00B.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.512000926Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2342024-02-28T22:12:48.790Z{"msg":"Repair extent 12","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.512026182Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
2352024-02-28T22:12:48.791Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/00C.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.512070826Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2362024-02-28T22:12:48.791Z{"msg":"eid:12 Found repair files: [\"00C\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.542643353Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2372024-02-28T22:12:48.791Z{"msg":"Verify extent 12 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.544146042Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2382024-02-28T22:12:48.791Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/00C.copy\" to \"/tmp/ds_test/clone/00/000/00C.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.544645979Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2392024-02-28T22:12:48.791Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/00C.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.5450479Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2402024-02-28T22:12:48.791Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/00C.replace\" to \"/tmp/ds_test/clone/00/000/00C.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.545171412Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2412024-02-28T22:12:48.791Z{"msg":"Repair extent 13","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.545256849Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
2422024-02-28T22:12:48.791Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/00D.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.545291764Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2432024-02-28T22:12:48.791Z{"msg":"eid:13 Found repair files: [\"00D\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.575826917Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2442024-02-28T22:12:48.791Z{"msg":"Verify extent 13 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.577372639Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2452024-02-28T22:12:48.791Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/00D.copy\" to \"/tmp/ds_test/clone/00/000/00D.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.577851639Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2462024-02-28T22:12:48.791Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/00D.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.578277146Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2472024-02-28T22:12:48.791Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/00D.replace\" to \"/tmp/ds_test/clone/00/000/00D.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.578326009Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2482024-02-28T22:12:48.791Z{"msg":"Repair extent 14","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.578343907Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
2492024-02-28T22:12:48.791Z{"msg":"Created copy dir \"/tmp/ds_test/clone/00/000/00E.copy\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.578359734Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2502024-02-28T22:12:48.791Z{"msg":"eid:14 Found repair files: [\"00E\"]","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.609154949Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2512024-02-28T22:12:48.791Z{"msg":"Verify extent 14 still ready for copy","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.610598397Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2522024-02-28T22:12:48.791Z{"msg":"1 repair files downloaded, move directory \"/tmp/ds_test/clone/00/000/00E.copy\" to \"/tmp/ds_test/clone/00/000/00E.replace\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.611095504Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2532024-02-28T22:12:48.792Z{"msg":"Copy files from \"/tmp/ds_test/clone/00/000/00E.replace\" in \"/tmp/ds_test/clone/00/000\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.611563255Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2542024-02-28T22:12:48.792Z{"msg":"Move directory \"/tmp/ds_test/clone/00/000/00E.replace\" to \"/tmp/ds_test/clone/00/000/00E.completed\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.611690207Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139}
2552024-02-28T22:12:48.792Z{"msg":"Region has been cloned","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.611726631Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1139,"task":"clone"}
2562024-02-28T22:12:48.792ZVerify clone using export
2572024-02-28T22:12:48.792Z{"msg":"Opened existing region file \"/tmp/ds_test/clone/region.json\"","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.629254136Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1140}
2582024-02-28T22:12:48.792Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.629466955Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1140}
2592024-02-28T22:12:48.792Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2024-02-28T22:12:47.629483442Z","hostname":"ip-10-150-1-188.us-west-2.compute.internal","pid":1140}
2602024-02-28T22:12:48.792ZExport total_size: 768000 Extent size:51200 Total Extents:15
2612024-02-28T22:12:48.792ZExporting from start_block: 0 count:300
2622024-02-28T22:12:48.792ZRead and wrote out 300 blocks
2632024-02-28T22:12:48.792ZStopping downstairs
2642024-02-28T22:12:48.792ZClone test passed
2652024-02-28T22:12:48.793Z
2662024-02-28T22:12:48.793Zreal 2.038824270
2672024-02-28T22:12:48.793Zuser 0.580970539
2682024-02-28T22:12:48.793Zsys 0.138843524
2692024-02-28T22:12:48.793Ztrap 0.000222032
2702024-02-28T22:12:48.793Ztflt 0.000549615
2712024-02-28T22:12:48.793Zdflt 0.002017251
2722024-02-28T22:12:48.793Zkflt 0.000788094
2732024-02-28T22:12:48.793Zlock 4.814542206
2742024-02-28T22:12:48.793Zslp 4.015265012
2752024-02-28T22:12:48.793Zlat 0.021428428
2762024-02-28T22:12:48.793Zstop 0.000410867
2772024-02-28T22:12:48.793Zprocess exited: duration 11135 ms, exit code 0
 
2782024-02-28T22:12:48.801Zfound 0 output files