make renameData() more defensive during overwrites (#19548)

instead upon any error in renameData(), we still
preserve the existing dataDir in some form for
recoverability in strange situations such as out
of disk space type errors.

Bonus: avoid running list and heal() instead allow
versions disparity to return the actual versions,
uuid to heal. Currently limit this to 100 versions
and lesser disparate objects.

an undo now reverts back the xl.meta from xl.meta.bkp
during overwrites on such flaky setups.

Bonus: Save N depth syscalls via skipping the parents
upon overwrites and versioned updates.

Flaky setup examples are stretch clusters with regular
packet drops etc, we need to add some defensive code
around to avoid dangling objects.
This commit is contained in:
Harshavardhana
2024-04-23 10:15:52 -07:00
committed by GitHub
parent ee1047bd52
commit 9693c382a8
22 changed files with 460 additions and 282 deletions

View File

@@ -19,6 +19,7 @@ package cmd
import (
"context"
"encoding/binary"
"errors"
"hash/crc32"
@@ -26,14 +27,17 @@ import (
)
// figure out the most commonVersions across disk that satisfies
// the 'writeQuorum' this function returns '0' if quorum cannot
// the 'writeQuorum' this function returns "" if quorum cannot
// be achieved and disks have too many inconsistent versions.
func reduceCommonVersions(diskVersions []uint64, writeQuorum int) (commonVersions uint64) {
func reduceCommonVersions(diskVersions [][]byte, writeQuorum int) (versions []byte) {
diskVersionsCount := make(map[uint64]int)
for _, versions := range diskVersions {
diskVersionsCount[versions]++
if len(versions) > 0 {
diskVersionsCount[binary.BigEndian.Uint64(versions)]++
}
}
var commonVersions uint64
max := 0
for versions, count := range diskVersionsCount {
if max < count {
@@ -43,10 +47,38 @@ func reduceCommonVersions(diskVersions []uint64, writeQuorum int) (commonVersion
}
if max >= writeQuorum {
return commonVersions
for _, versions := range diskVersions {
if binary.BigEndian.Uint64(versions) == commonVersions {
return versions
}
}
}
return 0
return []byte{}
}
// figure out the most commonVersions across disk that satisfies
// the 'writeQuorum' this function returns '0' if quorum cannot
// be achieved and disks have too many inconsistent versions.
func reduceCommonDataDir(dataDirs []string, writeQuorum int) (dataDir string) {
dataDirsCount := make(map[string]int)
for _, ddir := range dataDirs {
dataDirsCount[ddir]++
}
max := 0
for ddir, count := range dataDirsCount {
if max < count {
max = count
dataDir = ddir
}
}
if max >= writeQuorum {
return dataDir
}
return ""
}
// Returns number of errors that occurred the most (incl. nil) and the