Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

all: Remove lib/util package (#9049) 

Grab-bag packages are nasty, this cleans it up a little by splitting it
into topical packages sempahore, netutil, stringutil, structutil.
This commit is contained in:
Jakob Borg 2023-08-21 19:44:33 +02:00 committed by GitHub
parent 40b3b9ad15
commit acd767b30b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 414 additions and 385 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/api/api_test.go
View File

@ -44,8 +44,8 @@ import (
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syncthing/syncthing/lib/ur"
"github.com/syncthing/syncthing/lib/util"
"github.com/thejerf/suture/v4"
"golang.org/x/exp/slices"
)
var (
@ -1313,7 +1313,7 @@ func TestBrowse(t *testing.T) {
for _, tc := range cases {
ret := browseFiles(ffs, tc.current)
if !util.EqualStrings(ret, tc.returns) {
if !slices.Equal(ret, tc.returns) {
t.Errorf("browseFiles(%q) => %q, expected %q", tc.current, ret, tc.returns)
}
}

12
lib/api/confighandler.go
View File

@ -15,7 +15,7 @@ import (
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/structutil"
)
type configMuxBuilder struct {
@ -212,7 +212,7 @@ func (c *configMuxBuilder) registerDefaultFolder(path string) {
c.HandlerFunc(http.MethodPut, path, func(w http.ResponseWriter, r *http.Request) {
var cfg config.FolderConfiguration
util.SetDefaults(&cfg)
structutil.SetDefaults(&cfg)
c.adjustFolder(w, r, cfg, true)
})
@ -228,7 +228,7 @@ func (c *configMuxBuilder) registerDefaultDevice(path string) {
c.HandlerFunc(http.MethodPut, path, func(w http.ResponseWriter, r *http.Request) {
var cfg config.DeviceConfiguration
util.SetDefaults(&cfg)
structutil.SetDefaults(&cfg)
c.adjustDevice(w, r, cfg, true)
})
@ -266,7 +266,7 @@ func (c *configMuxBuilder) registerOptions(path string) {
c.HandlerFunc(http.MethodPut, path, func(w http.ResponseWriter, r *http.Request) {
var cfg config.OptionsConfiguration
util.SetDefaults(&cfg)
structutil.SetDefaults(&cfg)
c.adjustOptions(w, r, cfg)
})
@ -282,7 +282,7 @@ func (c *configMuxBuilder) registerLDAP(path string) {
c.HandlerFunc(http.MethodPut, path, func(w http.ResponseWriter, r *http.Request) {
var cfg config.LDAPConfiguration
util.SetDefaults(&cfg)
structutil.SetDefaults(&cfg)
c.adjustLDAP(w, r, cfg)
})
@ -298,7 +298,7 @@ func (c *configMuxBuilder) registerGUI(path string) {
c.HandlerFunc(http.MethodPut, path, func(w http.ResponseWriter, r *http.Request) {
var cfg config.GUIConfiguration
util.SetDefaults(&cfg)
structutil.SetDefaults(&cfg)
c.adjustGUI(w, r, cfg)
})

52
lib/config/config.go
View File

@ -16,14 +16,16 @@ import (
"net"
"net/url"
"os"
"reflect"
"sort"
"strconv"
"strings"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/netutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/structutil"
)
const (
@ -42,9 +44,9 @@ var (
// "consumer" of the configuration as we don't want these saved to the
// config.
DefaultListenAddresses = []string{
util.Address("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultTCPPort))),
netutil.AddressURL("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultTCPPort))),
"dynamic+https://relays.syncthing.net/endpoint",
util.Address("quic", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultQUICPort))),
netutil.AddressURL("quic", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultQUICPort))),
}
DefaultGUIPort = 8384
// DefaultDiscoveryServersV4 should be substituted when the configuration
@ -101,7 +103,7 @@ func New(myID protocol.DeviceID) Configuration {
cfg.Options.UnackedNotificationIDs = []string{"authenticationUserAndPassword"}
util.SetDefaults(&cfg)
structutil.SetDefaults(&cfg)
// Can't happen.
if err := cfg.prepare(myID); err != nil {
@ -127,9 +129,9 @@ func (cfg *Configuration) ProbeFreePorts() error {
cfg.Options.RawListenAddresses = []string{"default"}
} else {
cfg.Options.RawListenAddresses = []string{
util.Address("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))),
netutil.AddressURL("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))),
"dynamic+https://relays.syncthing.net/endpoint",
util.Address("quic", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))),
netutil.AddressURL("quic", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))),
}
}
@ -144,7 +146,7 @@ type xmlConfiguration struct {
func ReadXML(r io.Reader, myID protocol.DeviceID) (Configuration, int, error) {
var cfg xmlConfiguration
util.SetDefaults(&cfg)
structutil.SetDefaults(&cfg)
if err := xml.NewDecoder(r).Decode(&cfg); err != nil {
return Configuration{}, 0, err
@ -166,7 +168,7 @@ func ReadJSON(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
var cfg Configuration
util.SetDefaults(&cfg)
structutil.SetDefaults(&cfg)
if err := json.Unmarshal(bs, &cfg); err != nil {
return Configuration{}, err
@ -259,7 +261,7 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) error {
cfg.removeDeprecatedProtocols()
util.FillNilExceptDeprecated(cfg)
structutil.FillNilExceptDeprecated(cfg)
// TestIssue1750 relies on migrations happening after preparing options.
cfg.applyMigrations()
@ -636,7 +638,7 @@ func (defaults *Defaults) prepare(myID protocol.DeviceID, existingDevices map[pr
}
func ensureZeroForNodefault(empty interface{}, target interface{}) {
util.CopyMatchingTag(empty, target, "nodefault", func(v string) bool {
copyMatchingTag(empty, target, "nodefault", func(v string) bool {
if len(v) > 0 && v != "true" {
panic(fmt.Sprintf(`unexpected tag value: %s. expected untagged or "true"`, v))
}
@ -644,6 +646,36 @@ func ensureZeroForNodefault(empty interface{}, target interface{}) {
})
}
// copyMatchingTag copies fields tagged tag:"value" from "from" struct onto "to" struct.
func copyMatchingTag(from interface{}, to interface{}, tag string, shouldCopy func(value string) bool) {
fromStruct := reflect.ValueOf(from).Elem()
fromType := fromStruct.Type()
toStruct := reflect.ValueOf(to).Elem()
toType := toStruct.Type()
if fromType != toType {
panic(fmt.Sprintf("non equal types: %s != %s", fromType, toType))
}
for i := 0; i < toStruct.NumField(); i++ {
fromField := fromStruct.Field(i)
toField := toStruct.Field(i)
if !toField.CanSet() {
// Unexported fields
continue
}
structTag := toType.Field(i).Tag
v := structTag.Get(tag)
if shouldCopy(v) {
toField.Set(fromField)
}
}
}
func (i Ignores) Copy() Ignores {
out := Ignores{Lines: make([]string, len(i.Lines))}
copy(out.Lines, i.Lines)

58
lib/config/config_test.go
View File

@ -1597,3 +1597,61 @@ func handleFile(name string) {
fd.Write(origin)
fd.Close()
}
func TestCopyMatching(t *testing.T) {
type Nested struct {
A int
}
type Test struct {
CopyA int
CopyB []string
CopyC Nested
CopyD *Nested
NoCopy int `restart:"true"`
}
from := Test{
CopyA: 1,
CopyB: []string{"friend", "foe"},
CopyC: Nested{
A: 2,
},
CopyD: &Nested{
A: 3,
},
NoCopy: 4,
}
to := Test{
CopyA: 11,
CopyB: []string{"foot", "toe"},
CopyC: Nested{
A: 22,
},
CopyD: &Nested{
A: 33,
},
NoCopy: 44,
}
// Copy empty fields
copyMatchingTag(&from, &to, "restart", func(v string) bool {
return v != "true"
})
if to.CopyA != 1 {
t.Error("CopyA")
}
if len(to.CopyB) != 2 || to.CopyB[0] != "friend" || to.CopyB[1] != "foe" {
t.Error("CopyB")
}
if to.CopyC.A != 2 {
t.Error("CopyC")
}
if to.CopyD.A != 3 {
t.Error("CopyC")
}
if to.NoCopy != 44 {
t.Error("NoCopy")
}
}

3
lib/config/folderconfiguration.go
View File

@ -21,7 +21,6 @@ import (
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
)
var (
@ -244,7 +243,7 @@ func (f FolderConfiguration) RequiresRestartOnly() FolderConfiguration {
// copier, yet should not cause a restart.
blank := FolderConfiguration{}
util.CopyMatchingTag(&blank, &copy, "restart", func(v string) bool {
copyMatchingTag(&blank, &copy, "restart", func(v string) bool {
if len(v) > 0 && v != "false" {
panic(fmt.Sprintf(`unexpected tag value: %s. expected untagged or "false"`, v))
}

10
lib/config/migrations.go
View File

@ -17,8 +17,8 @@ import (
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/netutil"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/util"
)
// migrations is the set of config migration functions, with their target
@ -197,11 +197,11 @@ func migrateToConfigV24(cfg *Configuration) {
}
func migrateToConfigV23(cfg *Configuration) {
permBits := fs.FileMode(0777)
permBits := fs.FileMode(0o777)
if build.IsWindows {
// Windows has no umask so we must chose a safer set of bits to
// begin with.
permBits = 0700
permBits = 0o700
}
// Upgrade code remains hardcoded for .stfolder despite configurable
@ -391,14 +391,14 @@ func migrateToConfigV12(cfg *Configuration) {
// Change listen address schema
for i, addr := range cfg.Options.RawListenAddresses {
if len(addr) > 0 && !strings.HasPrefix(addr, "tcp://") {
cfg.Options.RawListenAddresses[i] = util.Address("tcp", addr)
cfg.Options.RawListenAddresses[i] = netutil.AddressURL("tcp", addr)
}
}
for i, device := range cfg.Devices {
for j, addr := range device.Addresses {
if addr != "dynamic" && addr != "" {
cfg.Devices[i].Addresses[j] = util.Address("tcp", addr)
cfg.Devices[i].Addresses[j] = netutil.AddressURL("tcp", addr)
}
}
}

17
lib/config/optionsconfiguration.go
View File

@ -12,7 +12,8 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/stringutil"
"github.com/syncthing/syncthing/lib/structutil"
)
func (opts OptionsConfiguration) Copy() OptionsConfiguration {
@ -29,10 +30,10 @@ func (opts OptionsConfiguration) Copy() OptionsConfiguration {
}
func (opts *OptionsConfiguration) prepare(guiPWIsSet bool) {
util.FillNilSlices(opts)
structutil.FillNilSlices(opts)
opts.RawListenAddresses = util.UniqueTrimmedStrings(opts.RawListenAddresses)
opts.RawGlobalAnnServers = util.UniqueTrimmedStrings(opts.RawGlobalAnnServers)
opts.RawListenAddresses = stringutil.UniqueTrimmedStrings(opts.RawListenAddresses)
opts.RawGlobalAnnServers = stringutil.UniqueTrimmedStrings(opts.RawGlobalAnnServers)
// Very short reconnection intervals are annoying
if opts.ReconnectIntervalS < 5 {
@ -71,7 +72,7 @@ func (opts *OptionsConfiguration) prepare(guiPWIsSet bool) {
func (opts OptionsConfiguration) RequiresRestartOnly() OptionsConfiguration {
optsCopy := opts
blank := OptionsConfiguration{}
util.CopyMatchingTag(&blank, &optsCopy, "restart", func(v string) bool {
copyMatchingTag(&blank, &optsCopy, "restart", func(v string) bool {
if len(v) > 0 && v != "true" {
panic(fmt.Sprintf(`unexpected tag value: %s. Expected untagged or "true"`, v))
}
@ -94,7 +95,7 @@ func (opts OptionsConfiguration) ListenAddresses() []string {
addresses = append(addresses, addr)
}
}
return util.UniqueTrimmedStrings(addresses)
return stringutil.UniqueTrimmedStrings(addresses)
}
func (opts OptionsConfiguration) StunServers() []string {
@ -116,7 +117,7 @@ func (opts OptionsConfiguration) StunServers() []string {
}
}
addresses = util.UniqueTrimmedStrings(addresses)
addresses = stringutil.UniqueTrimmedStrings(addresses)
return addresses
}
@ -135,7 +136,7 @@ func (opts OptionsConfiguration) GlobalDiscoveryServers() []string {
servers = append(servers, srv)
}
}
return util.UniqueTrimmedStrings(servers)
return stringutil.UniqueTrimmedStrings(servers)
}
func (opts OptionsConfiguration) MaxFolderConcurrency() int {

4
lib/config/size_test.go
View File

@ -10,7 +10,7 @@ import (
"testing"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/structutil"
)
type TestStruct struct {
@ -20,7 +20,7 @@ type TestStruct struct {
func TestSizeDefaults(t *testing.T) {
x := &TestStruct{}
util.SetDefaults(x)
structutil.SetDefaults(x)
if !x.Size.Percentage() {
t.Error("not percentage")

6
lib/config/versioningconfiguration.go
View File

@ -12,7 +12,7 @@ import (
"sort"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/structutil"
)
// internalVersioningConfiguration is used in XML serialization
@ -39,7 +39,7 @@ func (c VersioningConfiguration) Copy() VersioningConfiguration {
}
func (c *VersioningConfiguration) UnmarshalJSON(data []byte) error {
util.SetDefaults(c)
structutil.SetDefaults(c)
type noCustomUnmarshal VersioningConfiguration
ptr := (*noCustomUnmarshal)(c)
return json.Unmarshal(data, ptr)
@ -47,7 +47,7 @@ func (c *VersioningConfiguration) UnmarshalJSON(data []byte) error {
func (c *VersioningConfiguration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var intCfg internalVersioningConfiguration
util.SetDefaults(&intCfg)
structutil.SetDefaults(&intCfg)
if err := d.DecodeElement(&intCfg, &start); err != nil {
return err
}

19
lib/connections/service.go
View File

@ -23,6 +23,8 @@ import (
stdsync "sync"
"time"
"golang.org/x/exp/slices"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections/registry"
"github.com/syncthing/syncthing/lib/discover"
@ -30,9 +32,10 @@ import (
"github.com/syncthing/syncthing/lib/nat"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/stringutil"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
// Registers NAT service providers
_ "github.com/syncthing/syncthing/lib/pmp"
@ -582,7 +585,7 @@ func (s *service) dialDevices(ctx context.Context, now time.Time, cfg config.Con
// allowed additional number of connections (if limited).
numConns := 0
var numConnsMut stdsync.Mutex
dialSemaphore := util.NewSemaphore(dialMaxParallel)
dialSemaphore := semaphore.New(dialMaxParallel)
dialWG := new(stdsync.WaitGroup)
dialCtx, dialCancel := context.WithCancel(ctx)
defer func() {
@ -698,7 +701,7 @@ func (s *service) resolveDeviceAddrs(ctx context.Context, cfg config.DeviceConfi
addrs = append(addrs, addr)
}
}
return util.UniqueTrimmedStrings(addrs)
return stringutil.UniqueTrimmedStrings(addrs)
}
type lanChecker struct {
@ -875,7 +878,7 @@ func (s *service) checkAndSignalConnectLoopOnUpdatedDevices(from, to config.Conf
if oldDev, ok := oldDevices[dev.DeviceID]; !ok || oldDev.Paused {
s.dialNowDevices[dev.DeviceID] = struct{}{}
dial = true
} else if !util.EqualStrings(oldDev.Addresses, dev.Addresses) {
} else if !slices.Equal(oldDev.Addresses, dev.Addresses) {
dial = true
}
}
@ -905,7 +908,7 @@ func (s *service) AllAddresses() []string {
}
}
s.listenersMut.RUnlock()
return util.UniqueTrimmedStrings(addrs)
return stringutil.UniqueTrimmedStrings(addrs)
}
func (s *service) ExternalAddresses() []string {
@ -920,7 +923,7 @@ func (s *service) ExternalAddresses() []string {
}
}
s.listenersMut.RUnlock()
return util.UniqueTrimmedStrings(addrs)
return stringutil.UniqueTrimmedStrings(addrs)
}
func (s *service) ListenerStatus() map[string]ListenerStatusEntry {
@ -1079,7 +1082,7 @@ func IsAllowedNetwork(host string, allowed []string) bool {
return false
}
func (s *service) dialParallel(ctx context.Context, deviceID protocol.DeviceID, dialTargets []dialTarget, parentSema *util.Semaphore) (internalConn, bool) {
func (s *service) dialParallel(ctx context.Context, deviceID protocol.DeviceID, dialTargets []dialTarget, parentSema *semaphore.Semaphore) (internalConn, bool) {
// Group targets into buckets by priority
dialTargetBuckets := make(map[int][]dialTarget, len(dialTargets))
for _, tgt := range dialTargets {
@ -1095,7 +1098,7 @@ func (s *service) dialParallel(ctx context.Context, deviceID protocol.DeviceID,
// Sort the priorities so that we dial lowest first (which means highest...)
sort.Ints(priorities)
sema := util.MultiSemaphore{util.NewSemaphore(dialMaxParallelPerDevice), parentSema}
sema := semaphore.MultiSemaphore{semaphore.New(dialMaxParallelPerDevice), parentSema}
for _, prio := range priorities {
tgts := dialTargetBuckets[prio]
res := make(chan internalConn, len(tgts))

4
lib/db/lowlevel.go
View File

@ -23,9 +23,9 @@ import (
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/stringutil"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
"github.com/thejerf/suture/v4"
)
@ -1042,7 +1042,7 @@ func (db *Lowlevel) loadMetadataTracker(folder string) (*metadataTracker, error)
}
if age := time.Since(meta.Created()); age > db.recheckInterval {
l.Infof("Stored folder metadata for %q is %v old; recalculating", folder, util.NiceDurationString(age))
l.Infof("Stored folder metadata for %q is %v old; recalculating", folder, stringutil.NiceDurationString(age))
return db.getMetaAndCheck(folder)
}

6
lib/discover/manager.go
View File

@ -22,9 +22,9 @@ import (
"github.com/syncthing/syncthing/lib/connections/registry"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/stringutil"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
)
// The Manager aggregates results from multiple Finders. Each Finder has
@ -158,7 +158,7 @@ func (m *manager) Lookup(ctx context.Context, deviceID protocol.DeviceID) (addre
}
m.mut.RUnlock()
addresses = util.UniqueTrimmedStrings(addresses)
addresses = stringutil.UniqueTrimmedStrings(addresses)
sort.Strings(addresses)
l.Debugln("lookup results for", deviceID)
@ -223,7 +223,7 @@ func (m *manager) Cache() map[protocol.DeviceID]CacheEntry {
m.mut.RUnlock()
for k, v := range res {
v.Addresses = util.UniqueTrimmedStrings(v.Addresses)
v.Addresses = stringutil.UniqueTrimmedStrings(v.Addresses)
res[k] = v
}

9
lib/model/folder.go
View File

@ -24,10 +24,11 @@ import (
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/stats"
"github.com/syncthing/syncthing/lib/stringutil"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/watchaggregator"
)
@ -39,7 +40,7 @@ type folder struct {
stateTracker
config.FolderConfiguration
*stats.FolderStatisticsReference
ioLimiter *util.Semaphore
ioLimiter *semaphore.Semaphore
localFlags uint32
@ -95,7 +96,7 @@ type puller interface {
pull() (bool, error) // true when successful and should not be retried
}
func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, evLogger events.Logger, ioLimiter *util.Semaphore, ver versioner.Versioner) folder {
func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, evLogger events.Logger, ioLimiter *semaphore.Semaphore, ver versioner.Versioner) folder {
f := folder{
stateTracker: newStateTracker(cfg.ID, evLogger),
FolderConfiguration: cfg,
@ -426,7 +427,7 @@ func (f *folder) pull() (success bool, err error) {
// Pulling failed, try again later.
delay := f.pullPause + time.Since(startTime)
l.Infof("Folder %v isn't making sync progress - retrying in %v.", f.Description(), util.NiceDurationString(delay))
l.Infof("Folder %v isn't making sync progress - retrying in %v.", f.Description(), stringutil.NiceDurationString(delay))
f.pullFailTimer.Reset(delay)
return false, err

4
lib/model/folder_recvenc.go
View File

@ -16,7 +16,7 @@ import (
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/versioner"
)
@ -28,7 +28,7 @@ type receiveEncryptedFolder struct {
*sendReceiveFolder
}
func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service {
func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
f := &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)}
f.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
return f

4
lib/model/folder_recvonly.go
View File

@ -15,7 +15,7 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/versioner"
)
@ -57,7 +57,7 @@ type receiveOnlyFolder struct {
*sendReceiveFolder
}
func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service {
func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
sr := newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)
sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
return &receiveOnlyFolder{sr}

4
lib/model/folder_sendonly.go
View File

@ -12,7 +12,7 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/versioner"
)
@ -24,7 +24,7 @@ type sendOnlyFolder struct {
folder
}
func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service {
func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
f := &sendOnlyFolder{
folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, nil),
}

10
lib/model/folder_sendrecv.go
View File

@ -27,9 +27,9 @@ import (
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/weakhash"
)
@ -125,17 +125,17 @@ type sendReceiveFolder struct {
queue *jobQueue
blockPullReorderer blockPullReorderer
writeLimiter *util.Semaphore
writeLimiter *semaphore.Semaphore
tempPullErrors map[string]string // pull errors that might be just transient
}
func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service {
func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service {
f := &sendReceiveFolder{
folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, ver),
queue: newJobQueue(),
blockPullReorderer: newBlockPullReorderer(cfg.BlockPullOrder, model.id, cfg.DeviceIDs()),
writeLimiter: util.NewSemaphore(cfg.MaxConcurrentWrites),
writeLimiter: semaphore.New(cfg.MaxConcurrentWrites),
}
f.folder.puller = f
@ -1492,7 +1492,7 @@ func (*sendReceiveFolder) verifyBuffer(buf []byte, block protocol.BlockInfo) err
}
func (f *sendReceiveFolder) pullerRoutine(snap *db.Snapshot, in <-chan pullBlockState, out chan<- *sharedPullerState) {
requestLimiter := util.NewSemaphore(f.PullerMaxPendingKiB * 1024)
requestLimiter := semaphore.New(f.PullerMaxPendingKiB * 1024)
wg := sync.NewWaitGroup()
for state := range in {

24
lib/model/model.go
View File

@ -38,11 +38,11 @@ import (
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/stats"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/ur/contract"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
)
@ -136,10 +136,10 @@ type model struct {
shortID protocol.ShortID
// globalRequestLimiter limits the amount of data in concurrent incoming
// requests
globalRequestLimiter *util.Semaphore
globalRequestLimiter *semaphore.Semaphore
// folderIOLimiter limits the number of concurrent I/O heavy operations,
// such as scans and pulls.
folderIOLimiter *util.Semaphore
folderIOLimiter *semaphore.Semaphore
fatalChan chan error
started chan struct{}
keyGen *protocol.KeyGenerator
@ -160,7 +160,7 @@ type model struct {
// fields protected by pmut
pmut sync.RWMutex
conn map[protocol.DeviceID]protocol.Connection
connRequestLimiters map[protocol.DeviceID]*util.Semaphore
connRequestLimiters map[protocol.DeviceID]*semaphore.Semaphore
closed map[protocol.DeviceID]chan struct{}
helloMessages map[protocol.DeviceID]protocol.Hello
deviceDownloads map[protocol.DeviceID]*deviceDownloadState
@ -173,7 +173,7 @@ type model struct {
var _ config.Verifier = &model{}
type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, events.Logger, *util.Semaphore) service
type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, events.Logger, *semaphore.Semaphore) service
var folderFactories = make(map[config.FolderType]folderFactory)
@ -222,8 +222,8 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio
finder: db.NewBlockFinder(ldb),
progressEmitter: NewProgressEmitter(cfg, evLogger),
shortID: id.Short(),
globalRequestLimiter: util.NewSemaphore(1024 * cfg.Options().MaxConcurrentIncomingRequestKiB()),
folderIOLimiter: util.NewSemaphore(cfg.Options().MaxFolderConcurrency()),
globalRequestLimiter: semaphore.New(1024 * cfg.Options().MaxConcurrentIncomingRequestKiB()),
folderIOLimiter: semaphore.New(cfg.Options().MaxFolderConcurrency()),
fatalChan: make(chan error),
started: make(chan struct{}),
keyGen: keyGen,
@ -243,7 +243,7 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio
// fields protected by pmut
pmut: sync.NewRWMutex(),
conn: make(map[protocol.DeviceID]protocol.Connection),
connRequestLimiters: make(map[protocol.DeviceID]*util.Semaphore),
connRequestLimiters: make(map[protocol.DeviceID]*semaphore.Semaphore),
closed: make(map[protocol.DeviceID]chan struct{}),
helloMessages: make(map[protocol.DeviceID]protocol.Hello),
deviceDownloads: make(map[protocol.DeviceID]*deviceDownloadState),
@ -1966,8 +1966,8 @@ func (m *model) Request(conn protocol.Connection, folder, name string, _, size i
// skipping nil limiters, then returns a requestResponse of the given size.
// When the requestResponse is closed the limiters are given back the bytes,
// in reverse order.
func newLimitedRequestResponse(size int, limiters ...*util.Semaphore) *requestResponse {
multi := util.MultiSemaphore(limiters)
func newLimitedRequestResponse(size int, limiters ...*semaphore.Semaphore) *requestResponse {
multi := semaphore.MultiSemaphore(limiters)
multi.Take(size)
res := newRequestResponse(size)
@ -2261,9 +2261,9 @@ func (m *model) AddConnection(conn protocol.Connection, hello protocol.Hello) {
// 0: default, <0: no limiting
switch {
case device.MaxRequestKiB > 0:
m.connRequestLimiters[deviceID] = util.NewSemaphore(1024 * device.MaxRequestKiB)
m.connRequestLimiters[deviceID] = semaphore.New(1024 * device.MaxRequestKiB)
case device.MaxRequestKiB == 0:
m.connRequestLimiters[deviceID] = util.NewSemaphore(1024 * defaultPullerPendingKiB)
m.connRequestLimiters[deviceID] = semaphore.New(1024 * defaultPullerPendingKiB)
}
m.helloMessages[deviceID] = hello

16
lib/model/model_test.go
View File

@ -35,8 +35,8 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
protocolmocks "github.com/syncthing/syncthing/lib/protocol/mocks"
srand "github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/testutils"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/testutil"
"github.com/syncthing/syncthing/lib/versioner"
)
@ -2968,10 +2968,10 @@ func TestConnCloseOnRestart(t *testing.T) {
m := setupModel(t, w)
defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI())
br := &testutils.BlockingRW{}
nw := &testutils.NoopRW{}
br := &testutil.BlockingRW{}
nw := &testutil.NoopRW{}
ci := &protocolmocks.ConnectionInfo{}
m.AddConnection(protocol.NewConnection(device1, br, nw, testutils.NoopCloser{}, m, ci, protocol.CompressionNever, nil, m.keyGen), protocol.Hello{})
m.AddConnection(protocol.NewConnection(device1, br, nw, testutil.NoopCloser{}, m, ci, protocol.CompressionNever, nil, m.keyGen), protocol.Hello{})
m.pmut.RLock()
if len(m.closed) != 1 {
t.Fatalf("Expected just one conn (len(m.closed) == %v)", len(m.closed))
@ -3113,9 +3113,9 @@ func TestDeviceWasSeen(t *testing.T) {
}
func TestNewLimitedRequestResponse(t *testing.T) {
l0 := util.NewSemaphore(0)
l1 := util.NewSemaphore(1024)
l2 := (*util.Semaphore)(nil)
l0 := semaphore.New(0)
l1 := semaphore.New(1024)
l2 := (*semaphore.Semaphore)(nil)
// Should take 500 bytes from any non-unlimited non-nil limiters.
res := newLimitedRequestResponse(500, l0, l1, l2)

38
lib/model/queue_test.go
View File

@ -13,6 +13,7 @@ import (
"time"
"github.com/d4l3k/messagediff"
"golang.org/x/exp/slices"
)
func TestJobQueue(t *testing.T) {
@ -282,7 +283,6 @@ func BenchmarkJobQueuePushPopDone10k(b *testing.B) {
q.Done(n)
}
}
}
func TestQueuePagination(t *testing.T) {
@ -302,21 +302,21 @@ func TestQueuePagination(t *testing.T) {
progress, queued, skip = q.Jobs(1, 5)
if len(progress) != 0 || len(queued) != 5 || skip != 0 {
t.Error("Wrong length", len(progress), len(queued), 0)
} else if !equalStrings(queued, names[:5]) {
} else if !slices.Equal(queued, names[:5]) {
t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[:5])
}
progress, queued, skip = q.Jobs(2, 5)
if len(progress) != 0 || len(queued) != 5 || skip != 5 {
t.Error("Wrong length", len(progress), len(queued), 0)
} else if !equalStrings(queued, names[5:]) {
} else if !slices.Equal(queued, names[5:]) {
t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[5:])
}
progress, queued, skip = q.Jobs(2, 7)
if len(progress) != 0 || len(queued) != 3 || skip != 7 {
t.Error("Wrong length", len(progress), len(queued), 0)
} else if !equalStrings(queued, names[7:]) {
} else if !slices.Equal(queued, names[7:]) {
t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[7:])
}
@ -338,23 +338,23 @@ func TestQueuePagination(t *testing.T) {
progress, queued, skip = q.Jobs(1, 5)
if len(progress) != 1 || len(queued) != 4 || skip != 0 {
t.Error("Wrong length", len(progress), len(queued), 0)
} else if !equalStrings(progress, names[:1]) {
} else if !slices.Equal(progress, names[:1]) {
t.Errorf("Wrong elements in progress, got %v, expected %v", progress, names[:1])
} else if !equalStrings(queued, names[1:5]) {
} else if !slices.Equal(queued, names[1:5]) {
t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[1:5])
}
progress, queued, skip = q.Jobs(2, 5)
if len(progress) != 0 || len(queued) != 5 || skip != 5 {
t.Error("Wrong length", len(progress), len(queued), 0)
} else if !equalStrings(queued, names[5:]) {
} else if !slices.Equal(queued, names[5:]) {
t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[5:])
}
progress, queued, skip = q.Jobs(2, 7)
if len(progress) != 0 || len(queued) != 3 || skip != 7 {
t.Error("Wrong length", len(progress), len(queued), 0)
} else if !equalStrings(queued, names[7:]) {
} else if !slices.Equal(queued, names[7:]) {
t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[7:])
}
@ -378,25 +378,25 @@ func TestQueuePagination(t *testing.T) {
progress, queued, skip = q.Jobs(1, 5)
if len(progress) != 5 || len(queued) != 0 || skip != 0 {
t.Error("Wrong length", len(progress), len(queued), 0)
} else if !equalStrings(progress, names[:5]) {
} else if !slices.Equal(progress, names[:5]) {
t.Errorf("Wrong elements in progress, got %v, expected %v", progress, names[:5])
}
progress, queued, skip = q.Jobs(2, 5)
if len(progress) != 3 || len(queued) != 2 || skip != 5 {
t.Error("Wrong length", len(progress), len(queued), 0)
} else if !equalStrings(progress, names[5:8]) {
} else if !slices.Equal(progress, names[5:8]) {
t.Errorf("Wrong elements in progress, got %v, expected %v", progress, names[5:8])
} else if !equalStrings(queued, names[8:]) {
} else if !slices.Equal(queued, names[8:]) {
t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[8:])
}
progress, queued, skip = q.Jobs(2, 7)
if len(progress) != 1 || len(queued) != 2 || skip != 7 {
t.Error("Wrong length", len(progress), len(queued), 0)
} else if !equalStrings(progress, names[7:8]) {
} else if !slices.Equal(progress, names[7:8]) {
t.Errorf("Wrong elements in progress, got %v, expected %v", progress, names[7:8])
} else if !equalStrings(queued, names[8:]) {
} else if !slices.Equal(queued, names[8:]) {
t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[8:])
}
@ -405,15 +405,3 @@ func TestQueuePagination(t *testing.T) {
t.Error("Wrong length", len(progress), len(queued), 0)
}
}
func equalStrings(first, second []string) bool {
if len(first) != len(second) {
return false
}
for i := range first {
if first[i] != second[i] {
return false
}
}
return true
}

18
lib/netutil/netutil.go Normal file
View File

@ -0,0 +1,18 @@
// Copyright (C) 2023 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package netutil
import "net/url"
// Address constructs a URL from the given network and hostname.
func AddressURL(network, host string) string {
u := url.URL{
Scheme: network,
Host: host,
}
return u.String()
}

28
lib/netutil/netutil_test.go Normal file
View File

@ -0,0 +1,28 @@
// Copyright (C) 2023 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package netutil
import "testing"
func TestAddress(t *testing.T) {
tests := []struct {
network string
host string
result string
}{
{"tcp", "google.com", "tcp://google.com"},
{"foo", "google", "foo://google"},
{"123", "456", "123://456"},
}
for _, test := range tests {
result := AddressURL(test.network, test.host)
if result != test.result {
t.Errorf("%s != %s", result, test.result)
}
}
}

10
lib/pmp/pmp.go
View File

@ -19,7 +19,7 @@ import (
"github.com/syncthing/syncthing/lib/nat"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/svcutil"
)
func init() {
@ -28,7 +28,7 @@ func init() {
func Discover(ctx context.Context, renewal, timeout time.Duration) []nat.Device {
var ip net.IP
err := util.CallWithContext(ctx, func() error {
err := svcutil.CallWithContext(ctx, func() error {
var err error
ip, err = gateway.DiscoverGateway()
return err
@ -46,7 +46,7 @@ func Discover(ctx context.Context, renewal, timeout time.Duration) []nat.Device
c := natpmp.NewClientWithTimeout(ip, timeout)
// Try contacting the gateway, if it does not respond, assume it does not
// speak NAT-PMP.
err = util.CallWithContext(ctx, func() error {
err = svcutil.CallWithContext(ctx, func() error {
_, ierr := c.GetExternalAddress()
return ierr
})
@ -104,7 +104,7 @@ func (w *wrapper) AddPortMapping(ctx context.Context, protocol nat.Protocol, int
duration = w.renewal
}
var result *natpmp.AddPortMappingResult
err := util.CallWithContext(ctx, func() error {
err := svcutil.CallWithContext(ctx, func() error {
var err error
result, err = w.client.AddPortMapping(strings.ToLower(string(protocol)), internalPort, externalPort, int(duration/time.Second))
return err
@ -118,7 +118,7 @@ func (w *wrapper) AddPortMapping(ctx context.Context, protocol nat.Protocol, int
func (w *wrapper) GetExternalIPAddress(ctx context.Context) (net.IP, error) {
var result *natpmp.GetExternalAddressResult
err := util.CallWithContext(ctx, func() error {
err := svcutil.CallWithContext(ctx, func() error {
var err error
result, err = w.client.GetExternalAddress()
return err

6
lib/protocol/benchmark_test.go
View File

@ -10,7 +10,7 @@ import (
"testing"
"github.com/syncthing/syncthing/lib/dialer"
"github.com/syncthing/syncthing/lib/testutils"
"github.com/syncthing/syncthing/lib/testutil"
)
func BenchmarkRequestsRawTCP(b *testing.B) {
@ -60,9 +60,9 @@ func benchmarkRequestsTLS(b *testing.B, conn0, conn1 net.Conn) {
func benchmarkRequestsConnPair(b *testing.B, conn0, conn1 net.Conn) {
// Start up Connections on them
c0 := NewConnection(LocalDeviceID, conn0, conn0, testutils.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil, testKeyGen)
c0 := NewConnection(LocalDeviceID, conn0, conn0, testutil.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil, testKeyGen)
c0.Start()
c1 := NewConnection(LocalDeviceID, conn1, conn1, testutils.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil, testKeyGen)
c1 := NewConnection(LocalDeviceID, conn1, conn1, testutil.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil, testKeyGen)
c1.Start()
// Satisfy the assertions in the protocol by sending an initial cluster config

34
lib/protocol/protocol_test.go
View File

@ -19,7 +19,7 @@ import (
lz4 "github.com/pierrec/lz4/v4"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/testutils"
"github.com/syncthing/syncthing/lib/testutil"
)
var (
@ -32,10 +32,10 @@ func TestPing(t *testing.T) {
ar, aw := io.Pipe()
br, bw := io.Pipe()
c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutils.NoopCloser{}, newTestModel(), new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutil.NoopCloser{}, newTestModel(), new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
c0.Start()
defer closeAndWait(c0, ar, bw)
c1 := getRawConnection(NewConnection(c1ID, br, aw, testutils.NoopCloser{}, newTestModel(), new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
c1 := getRawConnection(NewConnection(c1ID, br, aw, testutil.NoopCloser{}, newTestModel(), new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
c1.Start()
defer closeAndWait(c1, ar, bw)
c0.ClusterConfig(ClusterConfig{})
@ -58,10 +58,10 @@ func TestClose(t *testing.T) {
ar, aw := io.Pipe()
br, bw := io.Pipe()
c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutils.NoopCloser{}, m0, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutil.NoopCloser{}, m0, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
c0.Start()
defer closeAndWait(c0, ar, bw)
c1 := NewConnection(c1ID, br, aw, testutils.NoopCloser{}, m1, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)
c1 := NewConnection(c1ID, br, aw, testutil.NoopCloser{}, m1, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)
c1.Start()
defer closeAndWait(c1, ar, bw)
c0.ClusterConfig(ClusterConfig{})
@ -102,8 +102,8 @@ func TestCloseOnBlockingSend(t *testing.T) {
m := newTestModel()
rw := testutils.NewBlockingRW()
c := getRawConnection(NewConnection(c0ID, rw, rw, testutils.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
rw := testutil.NewBlockingRW()
c := getRawConnection(NewConnection(c0ID, rw, rw, testutil.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
c.Start()
defer closeAndWait(c, rw)
@ -154,10 +154,10 @@ func TestCloseRace(t *testing.T) {
ar, aw := io.Pipe()
br, bw := io.Pipe()
c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutils.NoopCloser{}, m0, new(mockedConnectionInfo), CompressionNever, nil, testKeyGen))
c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutil.NoopCloser{}, m0, new(mockedConnectionInfo), CompressionNever, nil, testKeyGen))
c0.Start()
defer closeAndWait(c0, ar, bw)
c1 := NewConnection(c1ID, br, aw, testutils.NoopCloser{}, m1, new(mockedConnectionInfo), CompressionNever, nil, testKeyGen)
c1 := NewConnection(c1ID, br, aw, testutil.NoopCloser{}, m1, new(mockedConnectionInfo), CompressionNever, nil, testKeyGen)
c1.Start()
defer closeAndWait(c1, ar, bw)
c0.ClusterConfig(ClusterConfig{})
@ -193,8 +193,8 @@ func TestCloseRace(t *testing.T) {
func TestClusterConfigFirst(t *testing.T) {
m := newTestModel()
rw := testutils.NewBlockingRW()
c := getRawConnection(NewConnection(c0ID, rw, &testutils.NoopRW{}, testutils.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
rw := testutil.NewBlockingRW()
c := getRawConnection(NewConnection(c0ID, rw, &testutil.NoopRW{}, testutil.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
c.Start()
defer closeAndWait(c, rw)
@ -245,8 +245,8 @@ func TestCloseTimeout(t *testing.T) {
m := newTestModel()
rw := testutils.NewBlockingRW()
c := getRawConnection(NewConnection(c0ID, rw, rw, testutils.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
rw := testutil.NewBlockingRW()
c := getRawConnection(NewConnection(c0ID, rw, rw, testutil.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
c.Start()
defer closeAndWait(c, rw)
@ -898,8 +898,8 @@ func TestSha256OfEmptyBlock(t *testing.T) {
func TestClusterConfigAfterClose(t *testing.T) {
m := newTestModel()
rw := testutils.NewBlockingRW()
c := getRawConnection(NewConnection(c0ID, rw, rw, testutils.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
rw := testutil.NewBlockingRW()
c := getRawConnection(NewConnection(c0ID, rw, rw, testutil.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
c.Start()
defer closeAndWait(c, rw)
@ -922,8 +922,8 @@ func TestDispatcherToCloseDeadlock(t *testing.T) {
// Verify that we don't deadlock when calling Close() from within one of
// the model callbacks (ClusterConfig).
m := newTestModel()
rw := testutils.NewBlockingRW()
c := getRawConnection(NewConnection(c0ID, rw, &testutils.NoopRW{}, testutils.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
rw := testutil.NewBlockingRW()
c := getRawConnection(NewConnection(c0ID, rw, &testutil.NoopRW{}, testutil.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen))
m.ccFn = func(ClusterConfig) {
c.Close(errManual)
}

4
lib/util/semaphore.go → lib/semaphore/semaphore.go
View File

@ -4,7 +4,7 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package util
package semaphore
import (
"context"
@ -18,7 +18,7 @@ type Semaphore struct {
cond *sync.Cond
}
func NewSemaphore(max int) *Semaphore {
func New(max int) *Semaphore {
if max < 0 {
max = 0
}

24
lib/util/semaphore_test.go → lib/semaphore/semaphore_test.go
View File

@ -4,14 +4,16 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package util
package semaphore
import "testing"
func TestZeroByteSemaphore(_ *testing.T) {
func TestZeroByteSemaphore(t *testing.T) {
t.Parallel()
// A semaphore with zero capacity is just a no-op.
s := NewSemaphore(0)
s := New(0)
// None of these should block or panic
s.Take(123)
@ -20,9 +22,11 @@ func TestZeroByteSemaphore(_ *testing.T) {
}
func TestByteSemaphoreCapChangeUp(t *testing.T) {
t.Parallel()
// Waiting takes should unblock when the capacity increases
s := NewSemaphore(100)
s := New(100)
s.Take(75)
if s.available != 25 {
@ -43,9 +47,11 @@ func TestByteSemaphoreCapChangeUp(t *testing.T) {
}
func TestByteSemaphoreCapChangeDown1(t *testing.T) {
t.Parallel()
// Things should make sense when capacity is adjusted down
s := NewSemaphore(100)
s := New(100)
s.Take(75)
if s.available != 25 {
@ -64,9 +70,11 @@ func TestByteSemaphoreCapChangeDown1(t *testing.T) {
}
func TestByteSemaphoreCapChangeDown2(t *testing.T) {
t.Parallel()
// Things should make sense when capacity is adjusted down, different case
s := NewSemaphore(100)
s := New(100)
s.Take(75)
if s.available != 25 {
@ -85,9 +93,11 @@ func TestByteSemaphoreCapChangeDown2(t *testing.T) {
}
func TestByteSemaphoreGiveMore(t *testing.T) {
t.Parallel()
// We shouldn't end up with more available than we have capacity...
s := NewSemaphore(100)
s := New(100)
s.Take(150)
if s.available != 0 {

46
lib/stringutil/stringutil.go Normal file
View File

@ -0,0 +1,46 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package stringutil
import (
"strings"
"time"
)
// UniqueTrimmedStrings returns a list of all unique strings in ss,
// in the order in which they first appear in ss, after trimming away
// leading and trailing spaces.
func UniqueTrimmedStrings(ss []string) []string {
m := make(map[string]struct{}, len(ss))
us := make([]string, 0, len(ss))
for _, v := range ss {
v = strings.Trim(v, " ")
if _, ok := m[v]; ok {
continue
}
m[v] = struct{}{}
us = append(us, v)
}
return us
}
func NiceDurationString(d time.Duration) string {
switch {
case d > 24*time.Hour:
d = d.Round(time.Hour)
case d > time.Hour:
d = d.Round(time.Minute)
case d > time.Minute:
d = d.Round(time.Second)
case d > time.Second:
d = d.Round(time.Millisecond)
case d > time.Millisecond:
d = d.Round(time.Microsecond)
}
return d.String()
}

51
lib/stringutil/stringutil_test.go Normal file
View File

@ -0,0 +1,51 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package stringutil
import (
"testing"
)
func TestUniqueStrings(t *testing.T) {
tests := []struct {
input []string
expected []string
}{
{
[]string{"a", "b"},
[]string{"a", "b"},
},
{
[]string{"a", "a"},
[]string{"a"},
},
{
[]string{"a", "a", "a", "a"},
[]string{"a"},
},
{
nil,
nil,
},
{
[]string{" a ", " a ", "b ", " b"},
[]string{"a", "b"},
},
}
for _, test := range tests {
result := UniqueTrimmedStrings(test.input)
if len(result) != len(test.expected) {
t.Errorf("%s != %s", result, test.expected)
}
for i := range result {
if test.expected[i] != result[i] {
t.Errorf("%s != %s", result, test.expected)
}
}
}
}

118
lib/util/utils.go → lib/structutil/structutil.go
View File

@ -1,19 +1,15 @@
// Copyright (C) 2016 The Syncthing Authors.
// Copyright (C) 2023 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package util
package structutil
import (
"context"
"fmt"
"net/url"
"reflect"
"strconv"
"strings"
"time"
)
type defaultParser interface {
@ -21,7 +17,7 @@ type defaultParser interface {
}
// SetDefaults sets default values on a struct, based on the default annotation.
func SetDefaults(data interface{}) {
func SetDefaults(data any) {
s := reflect.ValueOf(data).Elem()
t := s.Type()
@ -86,63 +82,15 @@ func SetDefaults(data interface{}) {
}
}
// CopyMatchingTag copies fields tagged tag:"value" from "from" struct onto "to" struct.
func CopyMatchingTag(from interface{}, to interface{}, tag string, shouldCopy func(value string) bool) {
fromStruct := reflect.ValueOf(from).Elem()
fromType := fromStruct.Type()
toStruct := reflect.ValueOf(to).Elem()
toType := toStruct.Type()
if fromType != toType {
panic(fmt.Sprintf("non equal types: %s != %s", fromType, toType))
}
for i := 0; i < toStruct.NumField(); i++ {
fromField := fromStruct.Field(i)
toField := toStruct.Field(i)
if !toField.CanSet() {
// Unexported fields
continue
}
structTag := toType.Field(i).Tag
v := structTag.Get(tag)
if shouldCopy(v) {
toField.Set(fromField)
}
}
}
// UniqueTrimmedStrings returns a list of all unique strings in ss,
// in the order in which they first appear in ss, after trimming away
// leading and trailing spaces.
func UniqueTrimmedStrings(ss []string) []string {
var m = make(map[string]struct{}, len(ss))
var us = make([]string, 0, len(ss))
for _, v := range ss {
v = strings.Trim(v, " ")
if _, ok := m[v]; ok {
continue
}
m[v] = struct{}{}
us = append(us, v)
}
return us
}
func FillNilExceptDeprecated(data interface{}) {
func FillNilExceptDeprecated(data any) {
fillNil(data, true)
}
func FillNil(data interface{}) {
func FillNil(data any) {
fillNil(data, false)
}
func fillNil(data interface{}, skipDeprecated bool) {
func fillNil(data any, skipDeprecated bool) {
s := reflect.ValueOf(data).Elem()
t := s.Type()
for i := 0; i < s.NumField(); i++ {
@ -190,7 +138,7 @@ func fillNil(data interface{}, skipDeprecated bool) {
}
// FillNilSlices sets default value on slices that are still nil.
func FillNilSlices(data interface{}) error {
func FillNilSlices(data any) error {
s := reflect.ValueOf(data).Elem()
t := s.Type()
@ -220,55 +168,3 @@ func FillNilSlices(data interface{}) error {
}
return nil
}
// Address constructs a URL from the given network and hostname.
func Address(network, host string) string {
u := url.URL{
Scheme: network,
Host: host,
}
return u.String()
}
func CallWithContext(ctx context.Context, fn func() error) error {
var err error
done := make(chan struct{})
go func() {
err = fn()
close(done)
}()
select {
case <-done:
return err
case <-ctx.Done():
return ctx.Err()
}
}
func NiceDurationString(d time.Duration) string {
switch {
case d > 24*time.Hour:
d = d.Round(time.Hour)
case d > time.Hour:
d = d.Round(time.Minute)
case d > time.Minute:
d = d.Round(time.Second)
case d > time.Second:
d = d.Round(time.Millisecond)
case d > time.Millisecond:
d = d.Round(time.Microsecond)
}
return d.String()
}
func EqualStrings(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}

119
lib/util/utils_test.go → lib/structutil/structutil_test.go
View File

@ -4,7 +4,7 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package util
package structutil
import (
"testing"
@ -55,46 +55,6 @@ func TestSetDefaults(t *testing.T) {
}
}
func TestUniqueStrings(t *testing.T) {
tests := []struct {
input []string
expected []string
}{
{
[]string{"a", "b"},
[]string{"a", "b"},
},
{
[]string{"a", "a"},
[]string{"a"},
},
{
[]string{"a", "a", "a", "a"},
[]string{"a"},
},
{
nil,
nil,
},
{
[]string{" a ", " a ", "b ", " b"},
[]string{"a", "b"},
},
}
for _, test := range tests {
result := UniqueTrimmedStrings(test.input)
if len(result) != len(test.expected) {
t.Errorf("%s != %s", result, test.expected)
}
for i := range result {
if test.expected[i] != result[i] {
t.Errorf("%s != %s", result, test.expected)
}
}
}
}
func TestFillNillSlices(t *testing.T) {
// Nil
x := &struct {
@ -148,83 +108,6 @@ func TestFillNillSlices(t *testing.T) {
}
}
func TestAddress(t *testing.T) {
tests := []struct {
network string
host string
result string
}{
{"tcp", "google.com", "tcp://google.com"},
{"foo", "google", "foo://google"},
{"123", "456", "123://456"},
}
for _, test := range tests {
result := Address(test.network, test.host)
if result != test.result {
t.Errorf("%s != %s", result, test.result)
}
}
}
func TestCopyMatching(t *testing.T) {
type Nested struct {
A int
}
type Test struct {
CopyA int
CopyB []string
CopyC Nested
CopyD *Nested
NoCopy int `restart:"true"`
}
from := Test{
CopyA: 1,
CopyB: []string{"friend", "foe"},
CopyC: Nested{
A: 2,
},
CopyD: &Nested{
A: 3,
},
NoCopy: 4,
}
to := Test{
CopyA: 11,
CopyB: []string{"foot", "toe"},
CopyC: Nested{
A: 22,
},
CopyD: &Nested{
A: 33,
},
NoCopy: 44,
}
// Copy empty fields
CopyMatchingTag(&from, &to, "restart", func(v string) bool {
return v != "true"
})
if to.CopyA != 1 {
t.Error("CopyA")
}
if len(to.CopyB) != 2 || to.CopyB[0] != "friend" || to.CopyB[1] != "foe" {
t.Error("CopyB")
}
if to.CopyC.A != 2 {
t.Error("CopyC")
}
if to.CopyD.A != 3 {
t.Error("CopyC")
}
if to.NoCopy != 44 {
t.Error("NoCopy")
}
}
func TestFillNil(t *testing.T) {
type A struct {
Slice []int

4
lib/stun/stun.go
View File

@ -14,7 +14,7 @@ import (
"github.com/ccding/go-stun/stun"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/svcutil"
)
const stunRetryInterval = 5 * time.Minute
@ -159,7 +159,7 @@ func (s *Service) runStunForServer(ctx context.Context, addr string) {
var natType stun.NATType
var extAddr *stun.Host
err = util.CallWithContext(ctx, func() error {
err = svcutil.CallWithContext(ctx, func() error {
natType, extAddr, err = s.client.Discover()
return err
})

15
lib/svcutil/svcutil.go
View File

@ -223,3 +223,18 @@ func asNonContextError(ctx context.Context, err error) error {
}
return err
}
func CallWithContext(ctx context.Context, fn func() error) error {
var err error
done := make(chan struct{})
go func() {
err = fn()
close(done)
}()
select {
case <-done:
return err
case <-ctx.Done():
return ctx.Err()
}
}

3
lib/testutils/testutils.go → lib/testutil/testutil.go
View File

@ -4,7 +4,7 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package testutils
package testutil
import (
"errors"
@ -25,6 +25,7 @@ func NewBlockingRW() *BlockingRW {
closeOnce: sync.Once{},
}
}
func (rw *BlockingRW) Read(_ []byte) (int, error) {
<-rw.c
return 0, ErrClosed

4
lib/ur/contract/contract.go
View File

@ -14,7 +14,7 @@ import (
"strconv"
"time"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/structutil"
)
type Report struct {
@ -179,7 +179,7 @@ type Report struct {
func New() *Report {
r := &Report{}
util.FillNil(r)
structutil.FillNil(r)
return r
}

11
lib/versioner/util.go
View File

@ -19,7 +19,7 @@ import (
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/stringutil"
)
var (
@ -126,7 +126,6 @@ func retrieveVersions(fileSystem fs.Filesystem) (map[string][]FileVersion, error
return nil
})
if err != nil {
return nil, err
}
@ -153,7 +152,7 @@ func archiveFile(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, filePath
if err != nil {
if fs.IsNotExist(err) {
l.Debugln("creating versions dir")
err := dstFs.MkdirAll(".", 0755)
err := dstFs.MkdirAll(".", 0o755)
if err != nil {
return err
}
@ -166,7 +165,7 @@ func archiveFile(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, filePath
file := filepath.Base(filePath)
inFolderPath := filepath.Dir(filePath)
err = dstFs.MkdirAll(inFolderPath, 0755)
err = dstFs.MkdirAll(inFolderPath, 0o755)
if err != nil && !fs.IsExist(err) {
l.Debugln("archiving", filePath, err)
return err
@ -253,7 +252,7 @@ func restoreFile(method fs.CopyRangeMethod, src, dst fs.Filesystem, filePath str
return err
}
_ = dst.MkdirAll(filepath.Dir(filePath), 0755)
_ = dst.MkdirAll(filepath.Dir(filePath), 0o755)
err := osutil.RenameOrCopy(method, src, dst, sourceFile, filePath)
_ = dst.Chtimes(filePath, sourceMtime, sourceMtime)
return err
@ -285,7 +284,7 @@ func findAllVersions(fs fs.Filesystem, filePath string) []string {
l.Warnln("globbing:", err, "for", pattern)
return nil
}
versions = util.UniqueTrimmedStrings(versions)
versions = stringutil.UniqueTrimmedStrings(versions)
sort.Strings(versions)
return versions