Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cmd/syncthing: Add selectable sha256 package (fixes #3613, fixes #3614) 

This adds autodetection of the fastest hashing library on startup, thus
handling the performance regression. It also adds an environment
variable to control the selection, STHASHING=standard (Go standard
library version, avoids SIGILL crash when the minio library has bugs on
odd CPUs), STHASHING=minio (to force using the minio version) or unset
for the default autodetection.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3617
This commit is contained in:
Jakob Borg 2016-09-23 19:33:54 +00:00 committed by Audrius Butkevicius
parent 5f01afb7ea
commit d328e0fb75
9 changed files with 166 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
cmd/stdiscosrv/stdiscosrv Executable file
View File

Binary file not shown.

26
cmd/syncthing/main.go
View File

@ -41,6 +41,7 @@ import (
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/symlinks"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syncthing/syncthing/lib/upgrade"
@ -166,6 +167,11 @@ are mostly useful for developers. Use with care.
STNOUPGRADE Disable automatic upgrades.
STHASHING Select the SHA256 hashing package to use. Possible values
are "standard" for the Go standard library implementation,
"minio" for the github.com/minio/sha256-simd implementation,
and blank (the default) for auto detection.
GOMAXPROCS Set the maximum number of CPU cores to use. Defaults to all
available CPU cores.
@ -567,7 +573,9 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
l.Infoln(LongVersion)
l.Infoln("My ID:", myID)
printHashRate()
sha256.SelectAlgo()
sha256.Report()
// Emit the Starting event, now that we know who we are.
@ -840,22 +848,6 @@ func setupSignalHandling() {
}()
}
// printHashRate prints the hashing performance in MB/s, formatting it with
// appropriate precision for the value, i.e. 182 MB/s, 18 MB/s, 1.8 MB/s, 0.18
// MB/s.
func printHashRate() {
hashRate := cpuBench(3, 100*time.Millisecond)
decimals := 0
if hashRate < 1 {
decimals = 2
} else if hashRate < 10 {
decimals = 1
}
l.Infof("Single thread hash performance is ~%.*f MB/s", decimals, hashRate)
}
func loadConfig() (*config.Wrapper, error) {
cfgFile := locations[locConfigFile]
cfg, err := config.Load(cfgFile, myID)

16
cmd/syncthing/monitor.go
View File

@ -178,6 +178,22 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
if panicFd == nil {
dst.Write([]byte(line))
if strings.Contains(line, "SIGILL") {
l.Warnln(`
*******************************************************************************
* Crash due to illegal instruction detected. This is most likely due to a CPU *
* incompatibility with the high performance hashing package. Switching to the *
* standard hashing package instead. Please report this issue at: *
* *
* https://github.com/syncthing/syncthing/issues *
* *
* Include the details of your CPU. *
*******************************************************************************
`)
os.Setenv("STHASHING", "standard")
return
}
if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
panicFd, err = os.Create(timestampedLoc(locPanicLog))
if err != nil {

3
cmd/syncthing/usage_report.go
View File

@ -18,12 +18,11 @@ import (
"strings"
"time"
"github.com/minio/sha256-simd"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/dialer"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/thejerf/suture"
)

3
lib/protocol/bep_extensions.go
View File

@ -12,9 +12,8 @@ import (
"fmt"
"time"
"github.com/minio/sha256-simd"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
)
const (

2
lib/protocol/deviceid.go
View File

@ -11,7 +11,7 @@ import (
"regexp"
"strings"
"github.com/minio/sha256-simd"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/calmh/luhn"
)

3
lib/scanner/blocks.go
View File

@ -11,9 +11,8 @@ import (
"fmt"
"io"
"github.com/minio/sha256-simd"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
)
var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}

136
lib/sha256/sha256.go Normal file
View File

@ -0,0 +1,136 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package sha256
import (
"crypto/rand"
cryptoSha256 "crypto/sha256"
"fmt"
"hash"
"os"
"time"
minioSha256 "github.com/minio/sha256-simd"
"github.com/syncthing/syncthing/lib/logger"
)
var l = logger.DefaultLogger.NewFacility("sha256", "SHA256 hashing package")
const (
benchmarkingIterations = 3
benchmarkingDuration = 150 * time.Millisecond
defaultImpl = "crypto/sha256"
minioImpl = "minio/sha256-simd"
)
const (
BlockSize = cryptoSha256.BlockSize
Size = cryptoSha256.Size
)
// May be switched out for another implementation
var (
New = cryptoSha256.New
Sum256 = cryptoSha256.Sum256
)
var (
selectedImpl = defaultImpl
cryptoPerf float64
minioPerf float64
)
func SelectAlgo() {
switch os.Getenv("STHASHING") {
case "":
// When unset, probe for the fastest implementation.
benchmark()
if minioPerf > cryptoPerf {
selectMinio()
}
case "minio":
// When set to "minio", use that. Benchmark anyway to be able to
// present the difference.
benchmark()
selectMinio()
default:
// When set to anything else, such as "standard", use the default Go
// implementation. Benchmark that anyway, so we can report something
// useful in Report(). Make sure not to touch the minio
// implementation as it may be disabled for incompatibility reasons.
cryptoPerf = cpuBenchOnce(benchmarkingIterations*benchmarkingDuration, cryptoSha256.New)
}
}
// Report prints a line with the measured hash performance rates for the
// selected and alternate implementation.
func Report() {
var otherImpl string
var selectedRate, otherRate float64
switch selectedImpl {
case defaultImpl:
selectedRate = cryptoPerf
otherRate = minioPerf
otherImpl = minioImpl
case minioImpl:
selectedRate = minioPerf
otherRate = cryptoPerf
otherImpl = defaultImpl
}
l.Infof("Single thread hash performance is %s using %s (%s using %s).", formatRate(selectedRate), selectedImpl, formatRate(otherRate), otherImpl)
}
func selectMinio() {
New = minioSha256.New
Sum256 = minioSha256.Sum256
selectedImpl = minioImpl
}
func benchmark() {
// Interleave the tests to achieve some sort of fairness if the CPU is
// just in the process of spinning up to full speed.
for i := 0; i < benchmarkingIterations; i++ {
if perf := cpuBenchOnce(benchmarkingDuration, cryptoSha256.New); perf > cryptoPerf {
cryptoPerf = perf
}
if perf := cpuBenchOnce(benchmarkingDuration, minioSha256.New); perf > minioPerf {
minioPerf = perf
}
}
}
func cpuBenchOnce(duration time.Duration, newFn func() hash.Hash) float64 {
chunkSize := 100 * 1 << 10
h := newFn()
bs := make([]byte, chunkSize)
rand.Reader.Read(bs)
t0 := time.Now()
b := 0
for time.Since(t0) < duration {
h.Write(bs)
b += chunkSize
}
h.Sum(nil)
d := time.Since(t0)
return float64(int(float64(b)/d.Seconds()/(1<<20)*100)) / 100
}
func formatRate(rate float64) string {
decimals := 0
if rate < 1 {
decimals = 2
} else if rate < 10 {
decimals = 1
}
return fmt.Sprintf("%.*f MB/s", decimals, rate)
}

2
lib/signature/signature.go
View File

@ -20,7 +20,7 @@ import (
"io"
"math/big"
"github.com/minio/sha256-simd"
"github.com/syncthing/syncthing/lib/sha256"
)
// GenerateKeys returns a new key pair, with the private and public key