| 1 | | [ Dec 15 01:12:57 Disabled. ] |
| 2 | | [ Dec 15 01:12:57 Rereading configuration. ] |
| 3 | | [ Dec 15 01:13:01 Enabled. ] |
| 4 | | [ Dec 15 01:13:01 Rereading configuration. ] |
| 5 | | [ Dec 15 01:13:10 Executing start method ("/opt/oxide/lib/svc/manifest/crucible/pantry.sh"). ] |
| 6 | | + . /lib/svc/share/smf_include.sh |
| 7 | | ++ SMF_EXIT_OK=0 |
| 8 | | ++ SMF_EXIT_NODAEMON=94 |
| 9 | | ++ SMF_EXIT_ERR_FATAL=95 |
| 10 | | ++ SMF_EXIT_ERR_CONFIG=96 |
| 11 | | ++ SMF_EXIT_MON_DEGRADE=97 |
| 12 | | ++ SMF_EXIT_MON_OFFLINE=98 |
| 13 | | ++ SMF_EXIT_ERR_NOSMF=99 |
| 14 | | ++ SMF_EXIT_ERR_PERM=100 |
| 15 | | ++ svcprop -c -p config/datalink svc:/oxide/crucible/pantry:default |
| 16 | | + DATALINK=oxControlService10 |
| 17 | | ++ svcprop -c -p config/gateway svc:/oxide/crucible/pantry:default |
| 18 | | + GATEWAY=fd00:1122:3344:101::1 |
| 19 | | ++ svcprop -c -p config/listen_addr svc:/oxide/crucible/pantry:default |
| 20 | | + LISTEN_ADDR=fd00:1122:3344:101::10 |
| 21 | | ++ svcprop -c -p config/listen_port svc:/oxide/crucible/pantry:default |
| 22 | | + LISTEN_PORT=17000 |
| 23 | | + [[ oxControlService10 == unknown ]] |
| 24 | | + [[ fd00:1122:3344:101::1 == unknown ]] |
| 25 | | + ipadm delete-if oxControlService10 |
| 26 | | ipadm: Could not delete oxControlService10: Interface does not exist |
| 27 | | + true |
| 28 | | + ipadm create-if -t oxControlService10 |
| 29 | | + ipadm set-ifprop -t -p mtu=9000 -m ipv4 oxControlService10 |
| 30 | | + ipadm set-ifprop -t -p mtu=9000 -m ipv6 oxControlService10 |
| 31 | | + ipadm show-addr oxControlService10/ll |
| 32 | | ipadm: Address object not found |
| 33 | | + ipadm create-addr -t -T addrconf oxControlService10/ll |
| 34 | | + ipadm show-addr oxControlService10/omicron6 |
| 35 | | ipadm: Address object not found |
| 36 | | + ipadm create-addr -t -T static -a fd00:1122:3344:101::10 oxControlService10/omicron6 |
| 37 | | + route get -inet6 default -inet6 fd00:1122:3344:101::1 |
| 38 | | default: not in table |
| 39 | | + route add -inet6 default -inet6 fd00:1122:3344:101::1 |
| 40 | | add net default: gateway fd00:1122:3344:101::1 |
| 41 | | + args=('-l' "[$LISTEN_ADDR]:$LISTEN_PORT") |
| 42 | | + exec /opt/oxide/pantry/bin/crucible-pantry run -l '[fd00:1122:3344:101::10]:17000' |
| 43 | | note: configured to log to "/dev/stdout" |
| 44 | 2023-12-15T01:13:11.445Z | INFO | crucible-pantry (dropshot): listening
local_addr = [fd00:1122:3344:101::10]:17000
|
| 45 | 2023-12-15T01:13:11.487Z | INFO | crucible-pantry: listen IP: [fd00:1122:3344:101::10]:17000
|
| 46 | 2023-12-15T01:14:52.492Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:40181
|
| 47 | 2023-12-15T01:14:52.517Z | INFO | crucible-pantry (datafile): no entry exists for volume eee6274e-3d5d-4d9f-bae6-6d614ab39541, constructing...
|
| 48 | 2023-12-15T01:14:52.531Z | INFO | crucible-pantry (datafile): Upstairs starts
|
| 49 | 2023-12-15T01:14:52.538Z | INFO | crucible-pantry (datafile): Crucible Version: BuildInfo {
version: "0.0.1",
git_sha: "fab27994d0bd12725c17d6b478a9bfc2673ad6f4",
git_commit_timestamp: "2023-12-08T22:28:34.000000000Z",
git_branch: "main",
rustc_semver: "1.73.0",
rustc_channel: "stable",
rustc_host_triple: "x86_64-unknown-illumos",
rustc_commit_sha: "cc66ad468955717ab92600c770da8c1601a4ff33",
cargo_triple: "x86_64-unknown-illumos",
debug: false,
opt_level: 3,
}
|
| 50 | 2023-12-15T01:14:52.538Z | INFO | crucible-pantry (datafile): Upstairs <-> Downstairs Message Version: 5
|
| 51 | 2023-12-15T01:14:52.538Z | INFO | crucible-pantry (datafile): Using region definition RegionDefinition { block_size: 512, extent_size: Block { value: 131072, shift: 9 }, extent_count: 32, uuid: 00000000-0000-0000-0000-000000000000, encrypted: true, database_read_version: 1, database_write_version: 1 }
|
| 52 | 2023-12-15T01:14:52.558Z | INFO | crucible-pantry (datafile): Crucible eee6274e-3d5d-4d9f-bae6-6d614ab39541 has session id: 383e0509-ac5f-483d-9b63-c5a0e7fcba79
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 53 | 2023-12-15T01:14:52.558Z | INFO | crucible-pantry (datafile): Upstairs opts: Upstairs UUID: eee6274e-3d5d-4d9f-bae6-6d614ab39541, Targets: [[fd00:1122:3344:101::16]:19000, [fd00:1122:3344:101::19]:19000, [fd00:1122:3344:101::13]:19000], lossy: false, flush_timeout: None, key populated: true, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 54 | 2023-12-15T01:14:52.558Z | INFO | crucible-pantry (datafile): Crucible stats registered with UUID: eee6274e-3d5d-4d9f-bae6-6d614ab39541
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 55 | 2023-12-15T01:14:52.571Z | INFO | crucible-pantry (datafile): up_listen starts
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
task = up_listen
|
| 56 | 2023-12-15T01:14:52.571Z | INFO | crucible-pantry (datafile): Wait for all three downstairs to come online
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 57 | 2023-12-15T01:14:52.571Z | INFO | crucible-pantry (datafile): Flush timeout: 0.5
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 58 | 2023-12-15T01:14:52.571Z | INFO | crucible-pantry (datafile): [2] connecting to [fd00:1122:3344:101::13]:19000
looper = 2
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 59 | 2023-12-15T01:14:52.571Z | INFO | crucible-pantry (datafile): [1] connecting to [fd00:1122:3344:101::19]:19000
looper = 1
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 60 | 2023-12-15T01:14:52.571Z | INFO | crucible-pantry (datafile): [0] connecting to [fd00:1122:3344:101::16]:19000
looper = 0
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 61 | 2023-12-15T01:14:52.571Z | INFO | crucible-pantry (datafile): [2] eee6274e-3d5d-4d9f-bae6-6d614ab39541 looper connected
looper = 2
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 62 | 2023-12-15T01:14:52.573Z | INFO | crucible-pantry (datafile): [1] eee6274e-3d5d-4d9f-bae6-6d614ab39541 looper connected
looper = 1
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 63 | 2023-12-15T01:14:52.573Z | INFO | crucible-pantry (datafile): [0] eee6274e-3d5d-4d9f-bae6-6d614ab39541 looper connected
looper = 0
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 64 | 2023-12-15T01:14:52.573Z | INFO | crucible-pantry (datafile): [2] Proc runs for [fd00:1122:3344:101::13]:19000 in state New
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 65 | 2023-12-15T01:14:52.573Z | INFO | crucible-pantry (datafile): [1] Proc runs for [fd00:1122:3344:101::19]:19000 in state New
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 66 | 2023-12-15T01:14:52.573Z | INFO | crucible-pantry (datafile): volume eee6274e-3d5d-4d9f-bae6-6d614ab39541 constructed ok
|
| 67 | | The guest has requested activation |
| 68 | 2023-12-15T01:14:52.575Z | INFO | crucible-pantry (datafile): eee6274e-3d5d-4d9f-bae6-6d614ab39541 active request set
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 69 | 2023-12-15T01:14:52.577Z | INFO | crucible-pantry (datafile): [0] Proc runs for [fd00:1122:3344:101::16]:19000 in state New
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 70 | 2023-12-15T01:14:52.577Z | INFO | crucible-pantry (datafile): [2] eee6274e-3d5d-4d9f-bae6-6d614ab39541 (383e0509-ac5f-483d-9b63-c5a0e7fcba79) New New New ds_transition to WaitActive
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 71 | 2023-12-15T01:14:52.577Z | INFO | crucible-pantry (datafile): [2] Transition from New to WaitActive
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 72 | 2023-12-15T01:14:52.577Z | INFO | crucible-pantry (datafile): [2] client is_active_req TRUE, promote! session 383e0509-ac5f-483d-9b63-c5a0e7fcba79
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 73 | 2023-12-15T01:14:52.577Z | INFO | crucible-pantry (datafile): [0] eee6274e-3d5d-4d9f-bae6-6d614ab39541 (383e0509-ac5f-483d-9b63-c5a0e7fcba79) New New WaitActive ds_transition to WaitActive
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 74 | 2023-12-15T01:14:52.577Z | INFO | crucible-pantry (datafile): [0] Transition from New to WaitActive
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 75 | 2023-12-15T01:14:52.577Z | INFO | crucible-pantry (datafile): [0] client is_active_req TRUE, promote! session 383e0509-ac5f-483d-9b63-c5a0e7fcba79
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 76 | 2023-12-15T01:14:52.577Z | INFO | crucible-pantry (datafile): [1] eee6274e-3d5d-4d9f-bae6-6d614ab39541 (383e0509-ac5f-483d-9b63-c5a0e7fcba79) WaitActive New WaitActive ds_transition to WaitActive
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 77 | 2023-12-15T01:14:52.577Z | INFO | crucible-pantry (datafile): [1] Transition from New to WaitActive
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 78 | 2023-12-15T01:14:52.577Z | INFO | crucible-pantry (datafile): [1] client is_active_req TRUE, promote! session 383e0509-ac5f-483d-9b63-c5a0e7fcba79
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 79 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [2] downstairs client at [fd00:1122:3344:101::13]:19000 has UUID 81e0b7e6-3b1f-4fb8-a5fe-8da7614ef6e2
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 80 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [2] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 131072, shift: 9 }, extent_count: 32, uuid: 81e0b7e6-3b1f-4fb8-a5fe-8da7614ef6e2, encrypted: true, database_read_version: 1, database_write_version: 1 }
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 81 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): eee6274e-3d5d-4d9f-bae6-6d614ab39541 WaitActive WaitActive WaitActive
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 82 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [0] downstairs client at [fd00:1122:3344:101::16]:19000 has UUID 4e3c5ace-5778-43b5-bbf8-ea3089d319f4
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 83 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [0] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 131072, shift: 9 }, extent_count: 32, uuid: 4e3c5ace-5778-43b5-bbf8-ea3089d319f4, encrypted: true, database_read_version: 1, database_write_version: 1 }
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 84 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): eee6274e-3d5d-4d9f-bae6-6d614ab39541 WaitActive WaitActive WaitActive
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 85 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [1] downstairs client at [fd00:1122:3344:101::19]:19000 has UUID 33e518c6-a754-452c-8d74-0821ff294795
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 86 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [1] Got region def RegionDefinition { block_size: 512, extent_size: Block { value: 131072, shift: 9 }, extent_count: 32, uuid: 33e518c6-a754-452c-8d74-0821ff294795, encrypted: true, database_read_version: 1, database_write_version: 1 }
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 87 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): eee6274e-3d5d-4d9f-bae6-6d614ab39541 WaitActive WaitActive WaitActive
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 88 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [2] eee6274e-3d5d-4d9f-bae6-6d614ab39541 (383e0509-ac5f-483d-9b63-c5a0e7fcba79) WaitActive WaitActive WaitActive ds_transition to WaitQuorum
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 89 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [2] Transition from WaitActive to WaitQuorum
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 90 | 2023-12-15T01:14:52.578Z | WARN | crucible-pantry (datafile): [2] new RM replaced this: None
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 91 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [2] Starts reconcile loop
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 92 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [2] [fd00:1122:3344:101::13]:19000 task reports connection:true
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 93 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): eee6274e-3d5d-4d9f-bae6-6d614ab39541 WaitActive WaitActive WaitQuorum
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 94 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): Waiting for 2 more clients to be ready
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 95 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): Set check for repair
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 96 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [1] eee6274e-3d5d-4d9f-bae6-6d614ab39541 (383e0509-ac5f-483d-9b63-c5a0e7fcba79) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 97 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [1] Transition from WaitActive to WaitQuorum
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 98 | 2023-12-15T01:14:52.578Z | WARN | crucible-pantry (datafile): [1] new RM replaced this: None
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 99 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [1] Starts reconcile loop
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 100 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [1] [fd00:1122:3344:101::19]:19000 task reports connection:true
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 101 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): eee6274e-3d5d-4d9f-bae6-6d614ab39541 WaitActive WaitQuorum WaitQuorum
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 102 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): Waiting for 1 more clients to be ready
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 103 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): Set check for repair
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 104 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [0] eee6274e-3d5d-4d9f-bae6-6d614ab39541 (383e0509-ac5f-483d-9b63-c5a0e7fcba79) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 105 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [0] Transition from WaitActive to WaitQuorum
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 106 | 2023-12-15T01:14:52.578Z | WARN | crucible-pantry (datafile): [0] new RM replaced this: None
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 107 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [0] Starts reconcile loop
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 108 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): [0] [fd00:1122:3344:101::16]:19000 task reports connection:true
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 109 | 2023-12-15T01:14:52.578Z | INFO | crucible-pantry (datafile): eee6274e-3d5d-4d9f-bae6-6d614ab39541 WaitQuorum WaitQuorum WaitQuorum
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 110 | 2023-12-15T01:14:52.580Z | INFO | crucible-pantry (datafile): [0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 111 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 112 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 113 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 114 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 115 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 116 | | The guest has finished waiting for activation |
| 117 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 118 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 119 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 120 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): Max found gen is 1
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 121 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): Generation requested: 1 >= found:1
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 122 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): Next flush: 1
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 123 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): All extents match
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 124 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): No downstairs repair required
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 125 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): No initial repair work was required
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 126 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): Set Downstairs and Upstairs active
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 127 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): eee6274e-3d5d-4d9f-bae6-6d614ab39541 is now active with session: 383e0509-ac5f-483d-9b63-c5a0e7fcba79
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 128 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): eee6274e-3d5d-4d9f-bae6-6d614ab39541 Set Active after no repair
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 129 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): Notify all downstairs, region set compare is done.
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 130 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): Set check for repair
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 131 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [2] received reconcile message
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 132 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [2] All repairs completed, exit
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 133 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [0] received reconcile message
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 134 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [2] Starts cmd_loop
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 135 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [0] All repairs completed, exit
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 136 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [0] Starts cmd_loop
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 137 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [1] received reconcile message
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 138 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [1] All repairs completed, exit
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 139 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): [1] Starts cmd_loop
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 140 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): volume eee6274e-3d5d-4d9f-bae6-6d614ab39541 activated ok
|
| 141 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (datafile): volume eee6274e-3d5d-4d9f-bae6-6d614ab39541 constructed and inserted ok
|
| 142 | 2023-12-15T01:14:52.581Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 71723
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:40181
req_id = b4f4bd1f-2b23-42a1-922c-f4ba435b89cd
response_code = 200
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541
|
| 143 | 2023-12-15T01:14:53.581Z | INFO | crucible-pantry (datafile): Checking if live repair is needed
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 144 | 2023-12-15T01:14:53.581Z | INFO | crucible-pantry (datafile): No Live Repair required at this time
session_id = 383e0509-ac5f-483d-9b63-c5a0e7fcba79
|
| 145 | 2023-12-15T01:14:55.713Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:54695
|
| 146 | 2023-12-15T01:14:55.725Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 12633
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:54695
req_id = 2ad8c4e4-69bf-480c-ab04-bc5c399cc952
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 147 | 2023-12-15T01:14:56.260Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:42663
|
| 148 | 2023-12-15T01:14:56.265Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 4832
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:42663
req_id = 42f319a7-5186-40a4-9db4-c8223675dc4b
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 149 | 2023-12-15T01:14:56.395Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:50052
|
| 150 | 2023-12-15T01:14:56.402Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 6673
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:50052
req_id = fff4f0f8-76c9-409e-9984-8408646d9605
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 151 | 2023-12-15T01:14:56.946Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:63612
|
| 152 | 2023-12-15T01:14:56.951Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 4599
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:63612
req_id = e752fc5d-02f5-4962-a101-578ba66b7edc
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 153 | 2023-12-15T01:14:57.157Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:42657
|
| 154 | 2023-12-15T01:14:57.162Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 4390
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:42657
req_id = b5b03999-8488-47dc-a4fd-176915d0d4e8
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 155 | 2023-12-15T01:14:58.329Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:35367
|
| 156 | 2023-12-15T01:14:58.335Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:37496
|
| 157 | 2023-12-15T01:14:58.336Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 6861
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:35367
req_id = 55e95283-9e5b-4c12-9748-3faa8195f2d3
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 158 | 2023-12-15T01:14:58.339Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 3830
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:37496
req_id = bb6052d3-4792-4a08-9705-e152ec85648c
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 159 | 2023-12-15T01:14:58.633Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:34282
|
| 160 | 2023-12-15T01:14:58.638Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 4444
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:34282
req_id = 0f9d649b-5ea0-4e5a-8c1d-e5e1df444a48
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 161 | 2023-12-15T01:14:59.031Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:32792
|
| 162 | 2023-12-15T01:14:59.094Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 62940
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:32792
req_id = 9baf245e-e357-42f5-92e3-a855daafb92c
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 163 | 2023-12-15T01:14:59.223Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:38372
|
| 164 | 2023-12-15T01:14:59.228Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 4903
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:38372
req_id = 8770437b-fbcf-4d80-8148-38eb6cb4e6f0
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 165 | 2023-12-15T01:14:59.390Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:36423
|
| 166 | 2023-12-15T01:14:59.396Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 5443
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:36423
req_id = 219400a3-c082-466c-899d-8fcfdc4a349f
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 167 | 2023-12-15T01:14:59.523Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:50211
|
| 168 | 2023-12-15T01:14:59.529Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 6494
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:50211
req_id = 64c669d4-086f-476d-bf68-e467faf03b4f
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 169 | 2023-12-15T01:14:59.753Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:62813
|
| 170 | 2023-12-15T01:14:59.757Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 3786
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:62813
req_id = b641253f-8e22-4244-bc30-e67745ed48bd
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 171 | 2023-12-15T01:14:59.966Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:42781
|
| 172 | 2023-12-15T01:14:59.972Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 6296
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:42781
req_id = e4f72dab-dcb9-4911-8bb6-63ccf8f2c462
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 173 | 2023-12-15T01:15:00.045Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:54836
|
| 174 | 2023-12-15T01:15:00.050Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 5162
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:54836
req_id = d6009cb3-b8ce-4a42-85da-acf4420ed8e7
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 175 | 2023-12-15T01:15:00.066Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:47176
|
| 176 | 2023-12-15T01:15:00.070Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 4505
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:47176
req_id = 61285a29-dc79-45da-82f6-d18be3193809
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 177 | 2023-12-15T01:15:00.235Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:33647
|
| 178 | 2023-12-15T01:15:00.241Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 5328
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:33647
req_id = ea911bbc-28f1-4d18-989c-5409346df83e
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|
| 179 | 2023-12-15T01:15:00.339Z | INFO | crucible-pantry (dropshot): accepted connection
local_addr = [fd00:1122:3344:101::10]:17000
remote_addr = [fd00:1122:3344:101::a]:45264
|
| 180 | 2023-12-15T01:15:00.344Z | INFO | crucible-pantry (dropshot): request completed
latency_us = 5110
local_addr = [fd00:1122:3344:101::10]:17000
method = POST
remote_addr = [fd00:1122:3344:101::a]:45264
req_id = d8d1116d-f4a7-41a5-8da6-a6eb3dd5c286
response_code = 204
uri = /crucible/pantry/0/volume/eee6274e-3d5d-4d9f-bae6-6d614ab39541/bulk-write
|