Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

lib/protocol: Switch to a newer lz4 package (#8122)

This commit is contained in:
Jakob Borg 2022-01-17 18:52:43 +01:00 committed by GitHub
parent e700ef3208
commit b6d1e16b4e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 53 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -4,7 +4,6 @@ require (
github.com/AudriusButkevicius/pfilter v0.0.10
github.com/AudriusButkevicius/recli v0.0.6
github.com/alecthomas/kong v0.2.17
github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e
github.com/calmh/xdr v1.1.0
github.com/ccding/go-stun v0.1.3
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
@ -36,6 +35,7 @@ require (
github.com/minio/sha256-simd v1.0.0
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
github.com/oschwald/geoip2-golang v1.5.0
github.com/pierrec/lz4/v4 v4.1.12
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/common v0.30.0 // indirect

4
go.sum
View File

@ -60,8 +60,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e h1:2augTYh6E+XoNrrivZJBadpThP/dsvYKj0nzqfQ8tM4=
github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE=
@ -299,6 +297,8 @@ github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph
github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
github.com/pierrec/lz4/v4 v4.1.12 h1:44l88ehTZAUGW4VlO1QC4zkilL99M6Y9MXNwEs0uzP8=
github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

2
gui/default/syncthing/core/aboutModalView.html
View File

@ -37,7 +37,7 @@ Aaron Bieber, Adam Piggott, Adel Qalieh, Alan Pope, Alberto Donato, Alessandro G
<li><a href="https://github.com/AudriusButkevicius/go-nat-pmp">AudriusButkevicius/go-nat-pmp</a>, Copyright &copy; 2013 John Howard Palevich.</li>
<li><a href="https://github.com/AudriusButkevicius/recli">AudriusButkevicius/recli</a>, Copyright &copy; 2019 Audrius Butkevicius.</li>
<li><a href="https://github.com/beorn7/perks">beorn7/perks</a>, Copyright &copy; 2013 Blake Mizerany.</li>
<li><a href="https://github.com/bkaradzic/go-lz4">bkaradzic/go-lz4</a>, Copyright &copy; 2011-2012 Branimir Karadzic, 2013 Damian Gryski.</li>
<li><a href="https://github.com/pierrec/lz4">pierrec/lz4</a>, Copyright &copy; 2015 Pierre Curto.</li>
<li><a href="https://github.com/calmh/du">calmh/du</a>, Public domain.</li>
<li><a href="https://github.com/calmh/xdr">calmh/xdr</a>, Copyright &copy; 2014 Jakob Borg.</li>
<li><a href="https://github.com/chmduquesne/rollinghash">chmduquesne/rollinghash</a>, Copyright &copy; 2015 Christophe-Marie Duquesne.</li>

32
lib/protocol/protocol.go
View File

@ -23,7 +23,7 @@ import (
"sync"
"time"
lz4 "github.com/bkaradzic/go-lz4"
lz4 "github.com/pierrec/lz4/v4"
"github.com/pkg/errors"
)
@ -63,6 +63,10 @@ var sha256OfEmptyBlock = map[int][sha256.Size]byte{
16 << MiB: {0x8, 0xa, 0xcf, 0x35, 0xa5, 0x7, 0xac, 0x98, 0x49, 0xcf, 0xcb, 0xa4, 0x7d, 0xc2, 0xad, 0x83, 0xe0, 0x1b, 0x75, 0x66, 0x3a, 0x51, 0x62, 0x79, 0xc8, 0xb9, 0xd2, 0x43, 0xb7, 0x19, 0x64, 0x3e},
}
var (
errNotCompressible = errors.New("not compressible")
)
func init() {
for blockSize := MinBlockSize; blockSize <= MaxBlockSize; blockSize *= 2 {
BlockSizes = append(BlockSizes, blockSize)
@ -800,7 +804,7 @@ func (c *rawConnection) writeCompressedMessage(msg message, marshaled []byte, ov
}
cOverhead := 2 + hdrSize + 4
maxCompressed := cOverhead + lz4.CompressBound(len(marshaled))
maxCompressed := cOverhead + lz4.CompressBlockBound(len(marshaled))
buf := BufferPool.Get(maxCompressed)
defer BufferPool.Put(buf)
@ -1010,32 +1014,30 @@ func (c *rawConnection) Statistics() Statistics {
}
func lz4Compress(src, buf []byte) (int, error) {
compressed, err := lz4.Encode(buf, src)
// The compressed block is prefixed by the size of the uncompressed data.
binary.BigEndian.PutUint32(buf, uint32(len(src)))
n, err := lz4.CompressBlock(src, buf[4:], nil)
if err != nil {
return -1, err
}
if &compressed[0] != &buf[0] {
panic("bug: lz4.Compress allocated, which it must not (should use buffer pool)")
} else if len(src) > 0 && n == 0 {
return -1, errNotCompressible
}
binary.BigEndian.PutUint32(compressed, binary.LittleEndian.Uint32(compressed))
return len(compressed), nil
return n + 4, nil
}
func lz4Decompress(src []byte) ([]byte, error) {
size := binary.BigEndian.Uint32(src)
binary.LittleEndian.PutUint32(src, size)
var err error
buf := BufferPool.Get(int(size))
decoded, err := lz4.Decode(buf, src)
n, err := lz4.UncompressBlock(src[4:], buf)
if err != nil {
BufferPool.Put(buf)
return nil, err
}
if &decoded[0] != &buf[0] {
panic("bug: lz4.Decode allocated, which it must not (should use buffer pool)")
}
return decoded, nil
return buf[:n], nil
}
func newProtocolError(err error, msgContext string) error {

34
lib/protocol/protocol_test.go
View File

@ -17,7 +17,7 @@ import (
"testing/quick"
"time"
lz4 "github.com/bkaradzic/go-lz4"
lz4 "github.com/pierrec/lz4/v4"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/testutils"
)
@ -484,7 +484,7 @@ func TestLZ4Compression(t *testing.T) {
t.Fatal(err)
}
comp := make([]byte, lz4.CompressBound(dataLen))
comp := make([]byte, lz4.CompressBlockBound(dataLen))
compLen, err := lz4Compress(data, comp)
if err != nil {
t.Errorf("compressing %d bytes: %v", dataLen, err)
@ -506,6 +506,36 @@ func TestLZ4Compression(t *testing.T) {
}
}
func TestLZ4CompressionUpdate(t *testing.T) {
uncompressed := []byte("this is some arbitrary yet fairly compressible data")
// Compressed, as created by the LZ4 implementation in Syncthing 1.18.6 and earlier.
oldCompressed, _ := hex.DecodeString("00000033f0247468697320697320736f6d65206172626974726172792079657420666169726c7920636f6d707265737369626c652064617461")
// Verify that we can decompress
res, err := lz4Decompress(oldCompressed)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(uncompressed, res) {
t.Fatal("result does not match")
}
// Verify that our current compression is equivalent
buf := make([]byte, 128)
n, err := lz4Compress(uncompressed, buf)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(oldCompressed, buf[:n]) {
t.Logf("%x", oldCompressed)
t.Logf("%x", buf[:n])
t.Fatal("compression does not match")
}
}
func TestCheckFilename(t *testing.T) {
cases := []struct {
name string