diff --git a/lib/api/api_test.go b/lib/api/api_test.go index 6d1abfecb..be85ed6ee 100644 --- a/lib/api/api_test.go +++ b/lib/api/api_test.go @@ -44,8 +44,8 @@ import ( "github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/tlsutil" "github.com/syncthing/syncthing/lib/ur" - "github.com/syncthing/syncthing/lib/util" "github.com/thejerf/suture/v4" + "golang.org/x/exp/slices" ) var ( @@ -1313,7 +1313,7 @@ func TestBrowse(t *testing.T) { for _, tc := range cases { ret := browseFiles(ffs, tc.current) - if !util.EqualStrings(ret, tc.returns) { + if !slices.Equal(ret, tc.returns) { t.Errorf("browseFiles(%q) => %q, expected %q", tc.current, ret, tc.returns) } } diff --git a/lib/api/confighandler.go b/lib/api/confighandler.go index f26ba1c86..5abc1f055 100644 --- a/lib/api/confighandler.go +++ b/lib/api/confighandler.go @@ -15,7 +15,7 @@ import ( "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/protocol" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/structutil" ) type configMuxBuilder struct { @@ -212,7 +212,7 @@ func (c *configMuxBuilder) registerDefaultFolder(path string) { c.HandlerFunc(http.MethodPut, path, func(w http.ResponseWriter, r *http.Request) { var cfg config.FolderConfiguration - util.SetDefaults(&cfg) + structutil.SetDefaults(&cfg) c.adjustFolder(w, r, cfg, true) }) @@ -228,7 +228,7 @@ func (c *configMuxBuilder) registerDefaultDevice(path string) { c.HandlerFunc(http.MethodPut, path, func(w http.ResponseWriter, r *http.Request) { var cfg config.DeviceConfiguration - util.SetDefaults(&cfg) + structutil.SetDefaults(&cfg) c.adjustDevice(w, r, cfg, true) }) @@ -266,7 +266,7 @@ func (c *configMuxBuilder) registerOptions(path string) { c.HandlerFunc(http.MethodPut, path, func(w http.ResponseWriter, r *http.Request) { var cfg config.OptionsConfiguration - util.SetDefaults(&cfg) + structutil.SetDefaults(&cfg) c.adjustOptions(w, r, cfg) }) @@ -282,7 +282,7 @@ func (c *configMuxBuilder) registerLDAP(path string) { c.HandlerFunc(http.MethodPut, path, func(w http.ResponseWriter, r *http.Request) { var cfg config.LDAPConfiguration - util.SetDefaults(&cfg) + structutil.SetDefaults(&cfg) c.adjustLDAP(w, r, cfg) }) @@ -298,7 +298,7 @@ func (c *configMuxBuilder) registerGUI(path string) { c.HandlerFunc(http.MethodPut, path, func(w http.ResponseWriter, r *http.Request) { var cfg config.GUIConfiguration - util.SetDefaults(&cfg) + structutil.SetDefaults(&cfg) c.adjustGUI(w, r, cfg) }) diff --git a/lib/config/config.go b/lib/config/config.go index 8787cf8b5..4f02085fc 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -16,14 +16,16 @@ import ( "net" "net/url" "os" + "reflect" "sort" "strconv" "strings" "github.com/syncthing/syncthing/lib/build" "github.com/syncthing/syncthing/lib/fs" + "github.com/syncthing/syncthing/lib/netutil" "github.com/syncthing/syncthing/lib/protocol" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/structutil" ) const ( @@ -42,9 +44,9 @@ var ( // "consumer" of the configuration as we don't want these saved to the // config. DefaultListenAddresses = []string{ - util.Address("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultTCPPort))), + netutil.AddressURL("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultTCPPort))), "dynamic+https://relays.syncthing.net/endpoint", - util.Address("quic", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultQUICPort))), + netutil.AddressURL("quic", net.JoinHostPort("0.0.0.0", strconv.Itoa(DefaultQUICPort))), } DefaultGUIPort = 8384 // DefaultDiscoveryServersV4 should be substituted when the configuration @@ -101,7 +103,7 @@ func New(myID protocol.DeviceID) Configuration { cfg.Options.UnackedNotificationIDs = []string{"authenticationUserAndPassword"} - util.SetDefaults(&cfg) + structutil.SetDefaults(&cfg) // Can't happen. if err := cfg.prepare(myID); err != nil { @@ -127,9 +129,9 @@ func (cfg *Configuration) ProbeFreePorts() error { cfg.Options.RawListenAddresses = []string{"default"} } else { cfg.Options.RawListenAddresses = []string{ - util.Address("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))), + netutil.AddressURL("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))), "dynamic+https://relays.syncthing.net/endpoint", - util.Address("quic", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))), + netutil.AddressURL("quic", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))), } } @@ -144,7 +146,7 @@ type xmlConfiguration struct { func ReadXML(r io.Reader, myID protocol.DeviceID) (Configuration, int, error) { var cfg xmlConfiguration - util.SetDefaults(&cfg) + structutil.SetDefaults(&cfg) if err := xml.NewDecoder(r).Decode(&cfg); err != nil { return Configuration{}, 0, err @@ -166,7 +168,7 @@ func ReadJSON(r io.Reader, myID protocol.DeviceID) (Configuration, error) { var cfg Configuration - util.SetDefaults(&cfg) + structutil.SetDefaults(&cfg) if err := json.Unmarshal(bs, &cfg); err != nil { return Configuration{}, err @@ -259,7 +261,7 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) error { cfg.removeDeprecatedProtocols() - util.FillNilExceptDeprecated(cfg) + structutil.FillNilExceptDeprecated(cfg) // TestIssue1750 relies on migrations happening after preparing options. cfg.applyMigrations() @@ -636,7 +638,7 @@ func (defaults *Defaults) prepare(myID protocol.DeviceID, existingDevices map[pr } func ensureZeroForNodefault(empty interface{}, target interface{}) { - util.CopyMatchingTag(empty, target, "nodefault", func(v string) bool { + copyMatchingTag(empty, target, "nodefault", func(v string) bool { if len(v) > 0 && v != "true" { panic(fmt.Sprintf(`unexpected tag value: %s. expected untagged or "true"`, v)) } @@ -644,6 +646,36 @@ func ensureZeroForNodefault(empty interface{}, target interface{}) { }) } +// copyMatchingTag copies fields tagged tag:"value" from "from" struct onto "to" struct. +func copyMatchingTag(from interface{}, to interface{}, tag string, shouldCopy func(value string) bool) { + fromStruct := reflect.ValueOf(from).Elem() + fromType := fromStruct.Type() + + toStruct := reflect.ValueOf(to).Elem() + toType := toStruct.Type() + + if fromType != toType { + panic(fmt.Sprintf("non equal types: %s != %s", fromType, toType)) + } + + for i := 0; i < toStruct.NumField(); i++ { + fromField := fromStruct.Field(i) + toField := toStruct.Field(i) + + if !toField.CanSet() { + // Unexported fields + continue + } + + structTag := toType.Field(i).Tag + + v := structTag.Get(tag) + if shouldCopy(v) { + toField.Set(fromField) + } + } +} + func (i Ignores) Copy() Ignores { out := Ignores{Lines: make([]string, len(i.Lines))} copy(out.Lines, i.Lines) diff --git a/lib/config/config_test.go b/lib/config/config_test.go index e0d6ef22d..241cc2b59 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -1597,3 +1597,61 @@ func handleFile(name string) { fd.Write(origin) fd.Close() } + +func TestCopyMatching(t *testing.T) { + type Nested struct { + A int + } + type Test struct { + CopyA int + CopyB []string + CopyC Nested + CopyD *Nested + NoCopy int `restart:"true"` + } + + from := Test{ + CopyA: 1, + CopyB: []string{"friend", "foe"}, + CopyC: Nested{ + A: 2, + }, + CopyD: &Nested{ + A: 3, + }, + NoCopy: 4, + } + + to := Test{ + CopyA: 11, + CopyB: []string{"foot", "toe"}, + CopyC: Nested{ + A: 22, + }, + CopyD: &Nested{ + A: 33, + }, + NoCopy: 44, + } + + // Copy empty fields + copyMatchingTag(&from, &to, "restart", func(v string) bool { + return v != "true" + }) + + if to.CopyA != 1 { + t.Error("CopyA") + } + if len(to.CopyB) != 2 || to.CopyB[0] != "friend" || to.CopyB[1] != "foe" { + t.Error("CopyB") + } + if to.CopyC.A != 2 { + t.Error("CopyC") + } + if to.CopyD.A != 3 { + t.Error("CopyC") + } + if to.NoCopy != 44 { + t.Error("NoCopy") + } +} diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index d6e076727..3079c0f4c 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -21,7 +21,6 @@ import ( "github.com/syncthing/syncthing/lib/db" "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" - "github.com/syncthing/syncthing/lib/util" ) var ( @@ -244,7 +243,7 @@ func (f FolderConfiguration) RequiresRestartOnly() FolderConfiguration { // copier, yet should not cause a restart. blank := FolderConfiguration{} - util.CopyMatchingTag(&blank, ©, "restart", func(v string) bool { + copyMatchingTag(&blank, ©, "restart", func(v string) bool { if len(v) > 0 && v != "false" { panic(fmt.Sprintf(`unexpected tag value: %s. expected untagged or "false"`, v)) } diff --git a/lib/config/migrations.go b/lib/config/migrations.go index 5bafb4701..82f54d566 100644 --- a/lib/config/migrations.go +++ b/lib/config/migrations.go @@ -17,8 +17,8 @@ import ( "github.com/syncthing/syncthing/lib/build" "github.com/syncthing/syncthing/lib/fs" + "github.com/syncthing/syncthing/lib/netutil" "github.com/syncthing/syncthing/lib/upgrade" - "github.com/syncthing/syncthing/lib/util" ) // migrations is the set of config migration functions, with their target @@ -197,11 +197,11 @@ func migrateToConfigV24(cfg *Configuration) { } func migrateToConfigV23(cfg *Configuration) { - permBits := fs.FileMode(0777) + permBits := fs.FileMode(0o777) if build.IsWindows { // Windows has no umask so we must chose a safer set of bits to // begin with. - permBits = 0700 + permBits = 0o700 } // Upgrade code remains hardcoded for .stfolder despite configurable @@ -391,14 +391,14 @@ func migrateToConfigV12(cfg *Configuration) { // Change listen address schema for i, addr := range cfg.Options.RawListenAddresses { if len(addr) > 0 && !strings.HasPrefix(addr, "tcp://") { - cfg.Options.RawListenAddresses[i] = util.Address("tcp", addr) + cfg.Options.RawListenAddresses[i] = netutil.AddressURL("tcp", addr) } } for i, device := range cfg.Devices { for j, addr := range device.Addresses { if addr != "dynamic" && addr != "" { - cfg.Devices[i].Addresses[j] = util.Address("tcp", addr) + cfg.Devices[i].Addresses[j] = netutil.AddressURL("tcp", addr) } } } diff --git a/lib/config/optionsconfiguration.go b/lib/config/optionsconfiguration.go index 95d75416f..27e153790 100644 --- a/lib/config/optionsconfiguration.go +++ b/lib/config/optionsconfiguration.go @@ -12,7 +12,8 @@ import ( "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rand" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/stringutil" + "github.com/syncthing/syncthing/lib/structutil" ) func (opts OptionsConfiguration) Copy() OptionsConfiguration { @@ -29,10 +30,10 @@ func (opts OptionsConfiguration) Copy() OptionsConfiguration { } func (opts *OptionsConfiguration) prepare(guiPWIsSet bool) { - util.FillNilSlices(opts) + structutil.FillNilSlices(opts) - opts.RawListenAddresses = util.UniqueTrimmedStrings(opts.RawListenAddresses) - opts.RawGlobalAnnServers = util.UniqueTrimmedStrings(opts.RawGlobalAnnServers) + opts.RawListenAddresses = stringutil.UniqueTrimmedStrings(opts.RawListenAddresses) + opts.RawGlobalAnnServers = stringutil.UniqueTrimmedStrings(opts.RawGlobalAnnServers) // Very short reconnection intervals are annoying if opts.ReconnectIntervalS < 5 { @@ -71,7 +72,7 @@ func (opts *OptionsConfiguration) prepare(guiPWIsSet bool) { func (opts OptionsConfiguration) RequiresRestartOnly() OptionsConfiguration { optsCopy := opts blank := OptionsConfiguration{} - util.CopyMatchingTag(&blank, &optsCopy, "restart", func(v string) bool { + copyMatchingTag(&blank, &optsCopy, "restart", func(v string) bool { if len(v) > 0 && v != "true" { panic(fmt.Sprintf(`unexpected tag value: %s. Expected untagged or "true"`, v)) } @@ -94,7 +95,7 @@ func (opts OptionsConfiguration) ListenAddresses() []string { addresses = append(addresses, addr) } } - return util.UniqueTrimmedStrings(addresses) + return stringutil.UniqueTrimmedStrings(addresses) } func (opts OptionsConfiguration) StunServers() []string { @@ -116,7 +117,7 @@ func (opts OptionsConfiguration) StunServers() []string { } } - addresses = util.UniqueTrimmedStrings(addresses) + addresses = stringutil.UniqueTrimmedStrings(addresses) return addresses } @@ -135,7 +136,7 @@ func (opts OptionsConfiguration) GlobalDiscoveryServers() []string { servers = append(servers, srv) } } - return util.UniqueTrimmedStrings(servers) + return stringutil.UniqueTrimmedStrings(servers) } func (opts OptionsConfiguration) MaxFolderConcurrency() int { diff --git a/lib/config/size_test.go b/lib/config/size_test.go index d47bdde34..ab218abcc 100644 --- a/lib/config/size_test.go +++ b/lib/config/size_test.go @@ -10,7 +10,7 @@ import ( "testing" "github.com/syncthing/syncthing/lib/fs" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/structutil" ) type TestStruct struct { @@ -20,7 +20,7 @@ type TestStruct struct { func TestSizeDefaults(t *testing.T) { x := &TestStruct{} - util.SetDefaults(x) + structutil.SetDefaults(x) if !x.Size.Percentage() { t.Error("not percentage") diff --git a/lib/config/versioningconfiguration.go b/lib/config/versioningconfiguration.go index 2f7f87bc7..05355e746 100644 --- a/lib/config/versioningconfiguration.go +++ b/lib/config/versioningconfiguration.go @@ -12,7 +12,7 @@ import ( "sort" "github.com/syncthing/syncthing/lib/fs" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/structutil" ) // internalVersioningConfiguration is used in XML serialization @@ -39,7 +39,7 @@ func (c VersioningConfiguration) Copy() VersioningConfiguration { } func (c *VersioningConfiguration) UnmarshalJSON(data []byte) error { - util.SetDefaults(c) + structutil.SetDefaults(c) type noCustomUnmarshal VersioningConfiguration ptr := (*noCustomUnmarshal)(c) return json.Unmarshal(data, ptr) @@ -47,7 +47,7 @@ func (c *VersioningConfiguration) UnmarshalJSON(data []byte) error { func (c *VersioningConfiguration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { var intCfg internalVersioningConfiguration - util.SetDefaults(&intCfg) + structutil.SetDefaults(&intCfg) if err := d.DecodeElement(&intCfg, &start); err != nil { return err } diff --git a/lib/connections/service.go b/lib/connections/service.go index 55118f8fb..484fb94cc 100644 --- a/lib/connections/service.go +++ b/lib/connections/service.go @@ -23,6 +23,8 @@ import ( stdsync "sync" "time" + "golang.org/x/exp/slices" + "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/connections/registry" "github.com/syncthing/syncthing/lib/discover" @@ -30,9 +32,10 @@ import ( "github.com/syncthing/syncthing/lib/nat" "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/semaphore" + "github.com/syncthing/syncthing/lib/stringutil" "github.com/syncthing/syncthing/lib/svcutil" "github.com/syncthing/syncthing/lib/sync" - "github.com/syncthing/syncthing/lib/util" // Registers NAT service providers _ "github.com/syncthing/syncthing/lib/pmp" @@ -582,7 +585,7 @@ func (s *service) dialDevices(ctx context.Context, now time.Time, cfg config.Con // allowed additional number of connections (if limited). numConns := 0 var numConnsMut stdsync.Mutex - dialSemaphore := util.NewSemaphore(dialMaxParallel) + dialSemaphore := semaphore.New(dialMaxParallel) dialWG := new(stdsync.WaitGroup) dialCtx, dialCancel := context.WithCancel(ctx) defer func() { @@ -698,7 +701,7 @@ func (s *service) resolveDeviceAddrs(ctx context.Context, cfg config.DeviceConfi addrs = append(addrs, addr) } } - return util.UniqueTrimmedStrings(addrs) + return stringutil.UniqueTrimmedStrings(addrs) } type lanChecker struct { @@ -875,7 +878,7 @@ func (s *service) checkAndSignalConnectLoopOnUpdatedDevices(from, to config.Conf if oldDev, ok := oldDevices[dev.DeviceID]; !ok || oldDev.Paused { s.dialNowDevices[dev.DeviceID] = struct{}{} dial = true - } else if !util.EqualStrings(oldDev.Addresses, dev.Addresses) { + } else if !slices.Equal(oldDev.Addresses, dev.Addresses) { dial = true } } @@ -905,7 +908,7 @@ func (s *service) AllAddresses() []string { } } s.listenersMut.RUnlock() - return util.UniqueTrimmedStrings(addrs) + return stringutil.UniqueTrimmedStrings(addrs) } func (s *service) ExternalAddresses() []string { @@ -920,7 +923,7 @@ func (s *service) ExternalAddresses() []string { } } s.listenersMut.RUnlock() - return util.UniqueTrimmedStrings(addrs) + return stringutil.UniqueTrimmedStrings(addrs) } func (s *service) ListenerStatus() map[string]ListenerStatusEntry { @@ -1079,7 +1082,7 @@ func IsAllowedNetwork(host string, allowed []string) bool { return false } -func (s *service) dialParallel(ctx context.Context, deviceID protocol.DeviceID, dialTargets []dialTarget, parentSema *util.Semaphore) (internalConn, bool) { +func (s *service) dialParallel(ctx context.Context, deviceID protocol.DeviceID, dialTargets []dialTarget, parentSema *semaphore.Semaphore) (internalConn, bool) { // Group targets into buckets by priority dialTargetBuckets := make(map[int][]dialTarget, len(dialTargets)) for _, tgt := range dialTargets { @@ -1095,7 +1098,7 @@ func (s *service) dialParallel(ctx context.Context, deviceID protocol.DeviceID, // Sort the priorities so that we dial lowest first (which means highest...) sort.Ints(priorities) - sema := util.MultiSemaphore{util.NewSemaphore(dialMaxParallelPerDevice), parentSema} + sema := semaphore.MultiSemaphore{semaphore.New(dialMaxParallelPerDevice), parentSema} for _, prio := range priorities { tgts := dialTargetBuckets[prio] res := make(chan internalConn, len(tgts)) diff --git a/lib/db/lowlevel.go b/lib/db/lowlevel.go index 88792a36d..86bbaa31f 100644 --- a/lib/db/lowlevel.go +++ b/lib/db/lowlevel.go @@ -23,9 +23,9 @@ import ( "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/sha256" + "github.com/syncthing/syncthing/lib/stringutil" "github.com/syncthing/syncthing/lib/svcutil" "github.com/syncthing/syncthing/lib/sync" - "github.com/syncthing/syncthing/lib/util" "github.com/thejerf/suture/v4" ) @@ -1042,7 +1042,7 @@ func (db *Lowlevel) loadMetadataTracker(folder string) (*metadataTracker, error) } if age := time.Since(meta.Created()); age > db.recheckInterval { - l.Infof("Stored folder metadata for %q is %v old; recalculating", folder, util.NiceDurationString(age)) + l.Infof("Stored folder metadata for %q is %v old; recalculating", folder, stringutil.NiceDurationString(age)) return db.getMetaAndCheck(folder) } diff --git a/lib/discover/manager.go b/lib/discover/manager.go index efbdc05d2..4cec90dce 100644 --- a/lib/discover/manager.go +++ b/lib/discover/manager.go @@ -22,9 +22,9 @@ import ( "github.com/syncthing/syncthing/lib/connections/registry" "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/stringutil" "github.com/syncthing/syncthing/lib/svcutil" "github.com/syncthing/syncthing/lib/sync" - "github.com/syncthing/syncthing/lib/util" ) // The Manager aggregates results from multiple Finders. Each Finder has @@ -158,7 +158,7 @@ func (m *manager) Lookup(ctx context.Context, deviceID protocol.DeviceID) (addre } m.mut.RUnlock() - addresses = util.UniqueTrimmedStrings(addresses) + addresses = stringutil.UniqueTrimmedStrings(addresses) sort.Strings(addresses) l.Debugln("lookup results for", deviceID) @@ -223,7 +223,7 @@ func (m *manager) Cache() map[protocol.DeviceID]CacheEntry { m.mut.RUnlock() for k, v := range res { - v.Addresses = util.UniqueTrimmedStrings(v.Addresses) + v.Addresses = stringutil.UniqueTrimmedStrings(v.Addresses) res[k] = v } diff --git a/lib/model/folder.go b/lib/model/folder.go index b62281562..e69025d74 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -24,10 +24,11 @@ import ( "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" + "github.com/syncthing/syncthing/lib/semaphore" "github.com/syncthing/syncthing/lib/stats" + "github.com/syncthing/syncthing/lib/stringutil" "github.com/syncthing/syncthing/lib/svcutil" "github.com/syncthing/syncthing/lib/sync" - "github.com/syncthing/syncthing/lib/util" "github.com/syncthing/syncthing/lib/versioner" "github.com/syncthing/syncthing/lib/watchaggregator" ) @@ -39,7 +40,7 @@ type folder struct { stateTracker config.FolderConfiguration *stats.FolderStatisticsReference - ioLimiter *util.Semaphore + ioLimiter *semaphore.Semaphore localFlags uint32 @@ -95,7 +96,7 @@ type puller interface { pull() (bool, error) // true when successful and should not be retried } -func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, evLogger events.Logger, ioLimiter *util.Semaphore, ver versioner.Versioner) folder { +func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, evLogger events.Logger, ioLimiter *semaphore.Semaphore, ver versioner.Versioner) folder { f := folder{ stateTracker: newStateTracker(cfg.ID, evLogger), FolderConfiguration: cfg, @@ -426,7 +427,7 @@ func (f *folder) pull() (success bool, err error) { // Pulling failed, try again later. delay := f.pullPause + time.Since(startTime) - l.Infof("Folder %v isn't making sync progress - retrying in %v.", f.Description(), util.NiceDurationString(delay)) + l.Infof("Folder %v isn't making sync progress - retrying in %v.", f.Description(), stringutil.NiceDurationString(delay)) f.pullFailTimer.Reset(delay) return false, err diff --git a/lib/model/folder_recvenc.go b/lib/model/folder_recvenc.go index 7e2c5fda6..e10be6adf 100644 --- a/lib/model/folder_recvenc.go +++ b/lib/model/folder_recvenc.go @@ -16,7 +16,7 @@ import ( "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/ignore" "github.com/syncthing/syncthing/lib/protocol" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/semaphore" "github.com/syncthing/syncthing/lib/versioner" ) @@ -28,7 +28,7 @@ type receiveEncryptedFolder struct { *sendReceiveFolder } -func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service { +func newReceiveEncryptedFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service { f := &receiveEncryptedFolder{newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)} f.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files return f diff --git a/lib/model/folder_recvonly.go b/lib/model/folder_recvonly.go index 21f927cd4..ec76704f3 100644 --- a/lib/model/folder_recvonly.go +++ b/lib/model/folder_recvonly.go @@ -15,7 +15,7 @@ import ( "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/ignore" "github.com/syncthing/syncthing/lib/protocol" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/semaphore" "github.com/syncthing/syncthing/lib/versioner" ) @@ -57,7 +57,7 @@ type receiveOnlyFolder struct { *sendReceiveFolder } -func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service { +func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service { sr := newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder) sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files return &receiveOnlyFolder{sr} diff --git a/lib/model/folder_sendonly.go b/lib/model/folder_sendonly.go index 0e23a9f55..f1e59c467 100644 --- a/lib/model/folder_sendonly.go +++ b/lib/model/folder_sendonly.go @@ -12,7 +12,7 @@ import ( "github.com/syncthing/syncthing/lib/events" "github.com/syncthing/syncthing/lib/ignore" "github.com/syncthing/syncthing/lib/protocol" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/semaphore" "github.com/syncthing/syncthing/lib/versioner" ) @@ -24,7 +24,7 @@ type sendOnlyFolder struct { folder } -func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service { +func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service { f := &sendOnlyFolder{ folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, nil), } diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index bdc10fecf..5be363000 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -27,9 +27,9 @@ import ( "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" + "github.com/syncthing/syncthing/lib/semaphore" "github.com/syncthing/syncthing/lib/sha256" "github.com/syncthing/syncthing/lib/sync" - "github.com/syncthing/syncthing/lib/util" "github.com/syncthing/syncthing/lib/versioner" "github.com/syncthing/syncthing/lib/weakhash" ) @@ -125,17 +125,17 @@ type sendReceiveFolder struct { queue *jobQueue blockPullReorderer blockPullReorderer - writeLimiter *util.Semaphore + writeLimiter *semaphore.Semaphore tempPullErrors map[string]string // pull errors that might be just transient } -func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service { +func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *semaphore.Semaphore) service { f := &sendReceiveFolder{ folder: newFolder(model, fset, ignores, cfg, evLogger, ioLimiter, ver), queue: newJobQueue(), blockPullReorderer: newBlockPullReorderer(cfg.BlockPullOrder, model.id, cfg.DeviceIDs()), - writeLimiter: util.NewSemaphore(cfg.MaxConcurrentWrites), + writeLimiter: semaphore.New(cfg.MaxConcurrentWrites), } f.folder.puller = f @@ -1492,7 +1492,7 @@ func (*sendReceiveFolder) verifyBuffer(buf []byte, block protocol.BlockInfo) err } func (f *sendReceiveFolder) pullerRoutine(snap *db.Snapshot, in <-chan pullBlockState, out chan<- *sharedPullerState) { - requestLimiter := util.NewSemaphore(f.PullerMaxPendingKiB * 1024) + requestLimiter := semaphore.New(f.PullerMaxPendingKiB * 1024) wg := sync.NewWaitGroup() for state := range in { diff --git a/lib/model/model.go b/lib/model/model.go index 0fafd0d03..2ab5e8b22 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -38,11 +38,11 @@ import ( "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" + "github.com/syncthing/syncthing/lib/semaphore" "github.com/syncthing/syncthing/lib/stats" "github.com/syncthing/syncthing/lib/svcutil" "github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/ur/contract" - "github.com/syncthing/syncthing/lib/util" "github.com/syncthing/syncthing/lib/versioner" ) @@ -136,10 +136,10 @@ type model struct { shortID protocol.ShortID // globalRequestLimiter limits the amount of data in concurrent incoming // requests - globalRequestLimiter *util.Semaphore + globalRequestLimiter *semaphore.Semaphore // folderIOLimiter limits the number of concurrent I/O heavy operations, // such as scans and pulls. - folderIOLimiter *util.Semaphore + folderIOLimiter *semaphore.Semaphore fatalChan chan error started chan struct{} keyGen *protocol.KeyGenerator @@ -160,7 +160,7 @@ type model struct { // fields protected by pmut pmut sync.RWMutex conn map[protocol.DeviceID]protocol.Connection - connRequestLimiters map[protocol.DeviceID]*util.Semaphore + connRequestLimiters map[protocol.DeviceID]*semaphore.Semaphore closed map[protocol.DeviceID]chan struct{} helloMessages map[protocol.DeviceID]protocol.Hello deviceDownloads map[protocol.DeviceID]*deviceDownloadState @@ -173,7 +173,7 @@ type model struct { var _ config.Verifier = &model{} -type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, events.Logger, *util.Semaphore) service +type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, events.Logger, *semaphore.Semaphore) service var folderFactories = make(map[config.FolderType]folderFactory) @@ -222,8 +222,8 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio finder: db.NewBlockFinder(ldb), progressEmitter: NewProgressEmitter(cfg, evLogger), shortID: id.Short(), - globalRequestLimiter: util.NewSemaphore(1024 * cfg.Options().MaxConcurrentIncomingRequestKiB()), - folderIOLimiter: util.NewSemaphore(cfg.Options().MaxFolderConcurrency()), + globalRequestLimiter: semaphore.New(1024 * cfg.Options().MaxConcurrentIncomingRequestKiB()), + folderIOLimiter: semaphore.New(cfg.Options().MaxFolderConcurrency()), fatalChan: make(chan error), started: make(chan struct{}), keyGen: keyGen, @@ -243,7 +243,7 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio // fields protected by pmut pmut: sync.NewRWMutex(), conn: make(map[protocol.DeviceID]protocol.Connection), - connRequestLimiters: make(map[protocol.DeviceID]*util.Semaphore), + connRequestLimiters: make(map[protocol.DeviceID]*semaphore.Semaphore), closed: make(map[protocol.DeviceID]chan struct{}), helloMessages: make(map[protocol.DeviceID]protocol.Hello), deviceDownloads: make(map[protocol.DeviceID]*deviceDownloadState), @@ -1966,8 +1966,8 @@ func (m *model) Request(conn protocol.Connection, folder, name string, _, size i // skipping nil limiters, then returns a requestResponse of the given size. // When the requestResponse is closed the limiters are given back the bytes, // in reverse order. -func newLimitedRequestResponse(size int, limiters ...*util.Semaphore) *requestResponse { - multi := util.MultiSemaphore(limiters) +func newLimitedRequestResponse(size int, limiters ...*semaphore.Semaphore) *requestResponse { + multi := semaphore.MultiSemaphore(limiters) multi.Take(size) res := newRequestResponse(size) @@ -2261,9 +2261,9 @@ func (m *model) AddConnection(conn protocol.Connection, hello protocol.Hello) { // 0: default, <0: no limiting switch { case device.MaxRequestKiB > 0: - m.connRequestLimiters[deviceID] = util.NewSemaphore(1024 * device.MaxRequestKiB) + m.connRequestLimiters[deviceID] = semaphore.New(1024 * device.MaxRequestKiB) case device.MaxRequestKiB == 0: - m.connRequestLimiters[deviceID] = util.NewSemaphore(1024 * defaultPullerPendingKiB) + m.connRequestLimiters[deviceID] = semaphore.New(1024 * defaultPullerPendingKiB) } m.helloMessages[deviceID] = hello diff --git a/lib/model/model_test.go b/lib/model/model_test.go index daaa76fde..878239eaa 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -35,8 +35,8 @@ import ( "github.com/syncthing/syncthing/lib/protocol" protocolmocks "github.com/syncthing/syncthing/lib/protocol/mocks" srand "github.com/syncthing/syncthing/lib/rand" - "github.com/syncthing/syncthing/lib/testutils" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/semaphore" + "github.com/syncthing/syncthing/lib/testutil" "github.com/syncthing/syncthing/lib/versioner" ) @@ -2968,10 +2968,10 @@ func TestConnCloseOnRestart(t *testing.T) { m := setupModel(t, w) defer cleanupModelAndRemoveDir(m, fcfg.Filesystem(nil).URI()) - br := &testutils.BlockingRW{} - nw := &testutils.NoopRW{} + br := &testutil.BlockingRW{} + nw := &testutil.NoopRW{} ci := &protocolmocks.ConnectionInfo{} - m.AddConnection(protocol.NewConnection(device1, br, nw, testutils.NoopCloser{}, m, ci, protocol.CompressionNever, nil, m.keyGen), protocol.Hello{}) + m.AddConnection(protocol.NewConnection(device1, br, nw, testutil.NoopCloser{}, m, ci, protocol.CompressionNever, nil, m.keyGen), protocol.Hello{}) m.pmut.RLock() if len(m.closed) != 1 { t.Fatalf("Expected just one conn (len(m.closed) == %v)", len(m.closed)) @@ -3113,9 +3113,9 @@ func TestDeviceWasSeen(t *testing.T) { } func TestNewLimitedRequestResponse(t *testing.T) { - l0 := util.NewSemaphore(0) - l1 := util.NewSemaphore(1024) - l2 := (*util.Semaphore)(nil) + l0 := semaphore.New(0) + l1 := semaphore.New(1024) + l2 := (*semaphore.Semaphore)(nil) // Should take 500 bytes from any non-unlimited non-nil limiters. res := newLimitedRequestResponse(500, l0, l1, l2) diff --git a/lib/model/queue_test.go b/lib/model/queue_test.go index 9f49a683b..34ff67fd8 100644 --- a/lib/model/queue_test.go +++ b/lib/model/queue_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/d4l3k/messagediff" + "golang.org/x/exp/slices" ) func TestJobQueue(t *testing.T) { @@ -282,7 +283,6 @@ func BenchmarkJobQueuePushPopDone10k(b *testing.B) { q.Done(n) } } - } func TestQueuePagination(t *testing.T) { @@ -302,21 +302,21 @@ func TestQueuePagination(t *testing.T) { progress, queued, skip = q.Jobs(1, 5) if len(progress) != 0 || len(queued) != 5 || skip != 0 { t.Error("Wrong length", len(progress), len(queued), 0) - } else if !equalStrings(queued, names[:5]) { + } else if !slices.Equal(queued, names[:5]) { t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[:5]) } progress, queued, skip = q.Jobs(2, 5) if len(progress) != 0 || len(queued) != 5 || skip != 5 { t.Error("Wrong length", len(progress), len(queued), 0) - } else if !equalStrings(queued, names[5:]) { + } else if !slices.Equal(queued, names[5:]) { t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[5:]) } progress, queued, skip = q.Jobs(2, 7) if len(progress) != 0 || len(queued) != 3 || skip != 7 { t.Error("Wrong length", len(progress), len(queued), 0) - } else if !equalStrings(queued, names[7:]) { + } else if !slices.Equal(queued, names[7:]) { t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[7:]) } @@ -338,23 +338,23 @@ func TestQueuePagination(t *testing.T) { progress, queued, skip = q.Jobs(1, 5) if len(progress) != 1 || len(queued) != 4 || skip != 0 { t.Error("Wrong length", len(progress), len(queued), 0) - } else if !equalStrings(progress, names[:1]) { + } else if !slices.Equal(progress, names[:1]) { t.Errorf("Wrong elements in progress, got %v, expected %v", progress, names[:1]) - } else if !equalStrings(queued, names[1:5]) { + } else if !slices.Equal(queued, names[1:5]) { t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[1:5]) } progress, queued, skip = q.Jobs(2, 5) if len(progress) != 0 || len(queued) != 5 || skip != 5 { t.Error("Wrong length", len(progress), len(queued), 0) - } else if !equalStrings(queued, names[5:]) { + } else if !slices.Equal(queued, names[5:]) { t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[5:]) } progress, queued, skip = q.Jobs(2, 7) if len(progress) != 0 || len(queued) != 3 || skip != 7 { t.Error("Wrong length", len(progress), len(queued), 0) - } else if !equalStrings(queued, names[7:]) { + } else if !slices.Equal(queued, names[7:]) { t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[7:]) } @@ -378,25 +378,25 @@ func TestQueuePagination(t *testing.T) { progress, queued, skip = q.Jobs(1, 5) if len(progress) != 5 || len(queued) != 0 || skip != 0 { t.Error("Wrong length", len(progress), len(queued), 0) - } else if !equalStrings(progress, names[:5]) { + } else if !slices.Equal(progress, names[:5]) { t.Errorf("Wrong elements in progress, got %v, expected %v", progress, names[:5]) } progress, queued, skip = q.Jobs(2, 5) if len(progress) != 3 || len(queued) != 2 || skip != 5 { t.Error("Wrong length", len(progress), len(queued), 0) - } else if !equalStrings(progress, names[5:8]) { + } else if !slices.Equal(progress, names[5:8]) { t.Errorf("Wrong elements in progress, got %v, expected %v", progress, names[5:8]) - } else if !equalStrings(queued, names[8:]) { + } else if !slices.Equal(queued, names[8:]) { t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[8:]) } progress, queued, skip = q.Jobs(2, 7) if len(progress) != 1 || len(queued) != 2 || skip != 7 { t.Error("Wrong length", len(progress), len(queued), 0) - } else if !equalStrings(progress, names[7:8]) { + } else if !slices.Equal(progress, names[7:8]) { t.Errorf("Wrong elements in progress, got %v, expected %v", progress, names[7:8]) - } else if !equalStrings(queued, names[8:]) { + } else if !slices.Equal(queued, names[8:]) { t.Errorf("Wrong elements in queued, got %v, expected %v", queued, names[8:]) } @@ -405,15 +405,3 @@ func TestQueuePagination(t *testing.T) { t.Error("Wrong length", len(progress), len(queued), 0) } } - -func equalStrings(first, second []string) bool { - if len(first) != len(second) { - return false - } - for i := range first { - if first[i] != second[i] { - return false - } - } - return true -} diff --git a/lib/netutil/netutil.go b/lib/netutil/netutil.go new file mode 100644 index 000000000..d1b4a7f5b --- /dev/null +++ b/lib/netutil/netutil.go @@ -0,0 +1,18 @@ +// Copyright (C) 2023 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package netutil + +import "net/url" + +// Address constructs a URL from the given network and hostname. +func AddressURL(network, host string) string { + u := url.URL{ + Scheme: network, + Host: host, + } + return u.String() +} diff --git a/lib/netutil/netutil_test.go b/lib/netutil/netutil_test.go new file mode 100644 index 000000000..ccaebca09 --- /dev/null +++ b/lib/netutil/netutil_test.go @@ -0,0 +1,28 @@ +// Copyright (C) 2023 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package netutil + +import "testing" + +func TestAddress(t *testing.T) { + tests := []struct { + network string + host string + result string + }{ + {"tcp", "google.com", "tcp://google.com"}, + {"foo", "google", "foo://google"}, + {"123", "456", "123://456"}, + } + + for _, test := range tests { + result := AddressURL(test.network, test.host) + if result != test.result { + t.Errorf("%s != %s", result, test.result) + } + } +} diff --git a/lib/pmp/pmp.go b/lib/pmp/pmp.go index ac8823e7a..fe711b702 100644 --- a/lib/pmp/pmp.go +++ b/lib/pmp/pmp.go @@ -19,7 +19,7 @@ import ( "github.com/syncthing/syncthing/lib/nat" "github.com/syncthing/syncthing/lib/osutil" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/svcutil" ) func init() { @@ -28,7 +28,7 @@ func init() { func Discover(ctx context.Context, renewal, timeout time.Duration) []nat.Device { var ip net.IP - err := util.CallWithContext(ctx, func() error { + err := svcutil.CallWithContext(ctx, func() error { var err error ip, err = gateway.DiscoverGateway() return err @@ -46,7 +46,7 @@ func Discover(ctx context.Context, renewal, timeout time.Duration) []nat.Device c := natpmp.NewClientWithTimeout(ip, timeout) // Try contacting the gateway, if it does not respond, assume it does not // speak NAT-PMP. - err = util.CallWithContext(ctx, func() error { + err = svcutil.CallWithContext(ctx, func() error { _, ierr := c.GetExternalAddress() return ierr }) @@ -104,7 +104,7 @@ func (w *wrapper) AddPortMapping(ctx context.Context, protocol nat.Protocol, int duration = w.renewal } var result *natpmp.AddPortMappingResult - err := util.CallWithContext(ctx, func() error { + err := svcutil.CallWithContext(ctx, func() error { var err error result, err = w.client.AddPortMapping(strings.ToLower(string(protocol)), internalPort, externalPort, int(duration/time.Second)) return err @@ -118,7 +118,7 @@ func (w *wrapper) AddPortMapping(ctx context.Context, protocol nat.Protocol, int func (w *wrapper) GetExternalIPAddress(ctx context.Context) (net.IP, error) { var result *natpmp.GetExternalAddressResult - err := util.CallWithContext(ctx, func() error { + err := svcutil.CallWithContext(ctx, func() error { var err error result, err = w.client.GetExternalAddress() return err diff --git a/lib/protocol/benchmark_test.go b/lib/protocol/benchmark_test.go index 4fd07c407..588dee613 100644 --- a/lib/protocol/benchmark_test.go +++ b/lib/protocol/benchmark_test.go @@ -10,7 +10,7 @@ import ( "testing" "github.com/syncthing/syncthing/lib/dialer" - "github.com/syncthing/syncthing/lib/testutils" + "github.com/syncthing/syncthing/lib/testutil" ) func BenchmarkRequestsRawTCP(b *testing.B) { @@ -60,9 +60,9 @@ func benchmarkRequestsTLS(b *testing.B, conn0, conn1 net.Conn) { func benchmarkRequestsConnPair(b *testing.B, conn0, conn1 net.Conn) { // Start up Connections on them - c0 := NewConnection(LocalDeviceID, conn0, conn0, testutils.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil, testKeyGen) + c0 := NewConnection(LocalDeviceID, conn0, conn0, testutil.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil, testKeyGen) c0.Start() - c1 := NewConnection(LocalDeviceID, conn1, conn1, testutils.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil, testKeyGen) + c1 := NewConnection(LocalDeviceID, conn1, conn1, testutil.NoopCloser{}, new(fakeModel), new(mockedConnectionInfo), CompressionMetadata, nil, testKeyGen) c1.Start() // Satisfy the assertions in the protocol by sending an initial cluster config diff --git a/lib/protocol/protocol_test.go b/lib/protocol/protocol_test.go index 69f954aa3..fa751c1f7 100644 --- a/lib/protocol/protocol_test.go +++ b/lib/protocol/protocol_test.go @@ -19,7 +19,7 @@ import ( lz4 "github.com/pierrec/lz4/v4" "github.com/syncthing/syncthing/lib/build" "github.com/syncthing/syncthing/lib/rand" - "github.com/syncthing/syncthing/lib/testutils" + "github.com/syncthing/syncthing/lib/testutil" ) var ( @@ -32,10 +32,10 @@ func TestPing(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutils.NoopCloser{}, newTestModel(), new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) + c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutil.NoopCloser{}, newTestModel(), new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) c0.Start() defer closeAndWait(c0, ar, bw) - c1 := getRawConnection(NewConnection(c1ID, br, aw, testutils.NoopCloser{}, newTestModel(), new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) + c1 := getRawConnection(NewConnection(c1ID, br, aw, testutil.NoopCloser{}, newTestModel(), new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) c1.Start() defer closeAndWait(c1, ar, bw) c0.ClusterConfig(ClusterConfig{}) @@ -58,10 +58,10 @@ func TestClose(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutils.NoopCloser{}, m0, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) + c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutil.NoopCloser{}, m0, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) c0.Start() defer closeAndWait(c0, ar, bw) - c1 := NewConnection(c1ID, br, aw, testutils.NoopCloser{}, m1, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen) + c1 := NewConnection(c1ID, br, aw, testutil.NoopCloser{}, m1, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen) c1.Start() defer closeAndWait(c1, ar, bw) c0.ClusterConfig(ClusterConfig{}) @@ -102,8 +102,8 @@ func TestCloseOnBlockingSend(t *testing.T) { m := newTestModel() - rw := testutils.NewBlockingRW() - c := getRawConnection(NewConnection(c0ID, rw, rw, testutils.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) + rw := testutil.NewBlockingRW() + c := getRawConnection(NewConnection(c0ID, rw, rw, testutil.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) c.Start() defer closeAndWait(c, rw) @@ -154,10 +154,10 @@ func TestCloseRace(t *testing.T) { ar, aw := io.Pipe() br, bw := io.Pipe() - c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutils.NoopCloser{}, m0, new(mockedConnectionInfo), CompressionNever, nil, testKeyGen)) + c0 := getRawConnection(NewConnection(c0ID, ar, bw, testutil.NoopCloser{}, m0, new(mockedConnectionInfo), CompressionNever, nil, testKeyGen)) c0.Start() defer closeAndWait(c0, ar, bw) - c1 := NewConnection(c1ID, br, aw, testutils.NoopCloser{}, m1, new(mockedConnectionInfo), CompressionNever, nil, testKeyGen) + c1 := NewConnection(c1ID, br, aw, testutil.NoopCloser{}, m1, new(mockedConnectionInfo), CompressionNever, nil, testKeyGen) c1.Start() defer closeAndWait(c1, ar, bw) c0.ClusterConfig(ClusterConfig{}) @@ -193,8 +193,8 @@ func TestCloseRace(t *testing.T) { func TestClusterConfigFirst(t *testing.T) { m := newTestModel() - rw := testutils.NewBlockingRW() - c := getRawConnection(NewConnection(c0ID, rw, &testutils.NoopRW{}, testutils.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) + rw := testutil.NewBlockingRW() + c := getRawConnection(NewConnection(c0ID, rw, &testutil.NoopRW{}, testutil.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) c.Start() defer closeAndWait(c, rw) @@ -245,8 +245,8 @@ func TestCloseTimeout(t *testing.T) { m := newTestModel() - rw := testutils.NewBlockingRW() - c := getRawConnection(NewConnection(c0ID, rw, rw, testutils.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) + rw := testutil.NewBlockingRW() + c := getRawConnection(NewConnection(c0ID, rw, rw, testutil.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) c.Start() defer closeAndWait(c, rw) @@ -898,8 +898,8 @@ func TestSha256OfEmptyBlock(t *testing.T) { func TestClusterConfigAfterClose(t *testing.T) { m := newTestModel() - rw := testutils.NewBlockingRW() - c := getRawConnection(NewConnection(c0ID, rw, rw, testutils.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) + rw := testutil.NewBlockingRW() + c := getRawConnection(NewConnection(c0ID, rw, rw, testutil.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) c.Start() defer closeAndWait(c, rw) @@ -922,8 +922,8 @@ func TestDispatcherToCloseDeadlock(t *testing.T) { // Verify that we don't deadlock when calling Close() from within one of // the model callbacks (ClusterConfig). m := newTestModel() - rw := testutils.NewBlockingRW() - c := getRawConnection(NewConnection(c0ID, rw, &testutils.NoopRW{}, testutils.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) + rw := testutil.NewBlockingRW() + c := getRawConnection(NewConnection(c0ID, rw, &testutil.NoopRW{}, testutil.NoopCloser{}, m, new(mockedConnectionInfo), CompressionAlways, nil, testKeyGen)) m.ccFn = func(ClusterConfig) { c.Close(errManual) } diff --git a/lib/util/semaphore.go b/lib/semaphore/semaphore.go similarity index 98% rename from lib/util/semaphore.go rename to lib/semaphore/semaphore.go index 276234ed7..63bd6d277 100644 --- a/lib/util/semaphore.go +++ b/lib/semaphore/semaphore.go @@ -4,7 +4,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -package util +package semaphore import ( "context" @@ -18,7 +18,7 @@ type Semaphore struct { cond *sync.Cond } -func NewSemaphore(max int) *Semaphore { +func New(max int) *Semaphore { if max < 0 { max = 0 } diff --git a/lib/util/semaphore_test.go b/lib/semaphore/semaphore_test.go similarity index 90% rename from lib/util/semaphore_test.go rename to lib/semaphore/semaphore_test.go index dd1e56f4a..7ec047e36 100644 --- a/lib/util/semaphore_test.go +++ b/lib/semaphore/semaphore_test.go @@ -4,14 +4,16 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -package util +package semaphore import "testing" -func TestZeroByteSemaphore(_ *testing.T) { +func TestZeroByteSemaphore(t *testing.T) { + t.Parallel() + // A semaphore with zero capacity is just a no-op. - s := NewSemaphore(0) + s := New(0) // None of these should block or panic s.Take(123) @@ -20,9 +22,11 @@ func TestZeroByteSemaphore(_ *testing.T) { } func TestByteSemaphoreCapChangeUp(t *testing.T) { + t.Parallel() + // Waiting takes should unblock when the capacity increases - s := NewSemaphore(100) + s := New(100) s.Take(75) if s.available != 25 { @@ -43,9 +47,11 @@ func TestByteSemaphoreCapChangeUp(t *testing.T) { } func TestByteSemaphoreCapChangeDown1(t *testing.T) { + t.Parallel() + // Things should make sense when capacity is adjusted down - s := NewSemaphore(100) + s := New(100) s.Take(75) if s.available != 25 { @@ -64,9 +70,11 @@ func TestByteSemaphoreCapChangeDown1(t *testing.T) { } func TestByteSemaphoreCapChangeDown2(t *testing.T) { + t.Parallel() + // Things should make sense when capacity is adjusted down, different case - s := NewSemaphore(100) + s := New(100) s.Take(75) if s.available != 25 { @@ -85,9 +93,11 @@ func TestByteSemaphoreCapChangeDown2(t *testing.T) { } func TestByteSemaphoreGiveMore(t *testing.T) { + t.Parallel() + // We shouldn't end up with more available than we have capacity... - s := NewSemaphore(100) + s := New(100) s.Take(150) if s.available != 0 { diff --git a/lib/stringutil/stringutil.go b/lib/stringutil/stringutil.go new file mode 100644 index 000000000..fdcab2768 --- /dev/null +++ b/lib/stringutil/stringutil.go @@ -0,0 +1,46 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package stringutil + +import ( + "strings" + "time" +) + +// UniqueTrimmedStrings returns a list of all unique strings in ss, +// in the order in which they first appear in ss, after trimming away +// leading and trailing spaces. +func UniqueTrimmedStrings(ss []string) []string { + m := make(map[string]struct{}, len(ss)) + us := make([]string, 0, len(ss)) + for _, v := range ss { + v = strings.Trim(v, " ") + if _, ok := m[v]; ok { + continue + } + m[v] = struct{}{} + us = append(us, v) + } + + return us +} + +func NiceDurationString(d time.Duration) string { + switch { + case d > 24*time.Hour: + d = d.Round(time.Hour) + case d > time.Hour: + d = d.Round(time.Minute) + case d > time.Minute: + d = d.Round(time.Second) + case d > time.Second: + d = d.Round(time.Millisecond) + case d > time.Millisecond: + d = d.Round(time.Microsecond) + } + return d.String() +} diff --git a/lib/stringutil/stringutil_test.go b/lib/stringutil/stringutil_test.go new file mode 100644 index 000000000..3ccac2281 --- /dev/null +++ b/lib/stringutil/stringutil_test.go @@ -0,0 +1,51 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package stringutil + +import ( + "testing" +) + +func TestUniqueStrings(t *testing.T) { + tests := []struct { + input []string + expected []string + }{ + { + []string{"a", "b"}, + []string{"a", "b"}, + }, + { + []string{"a", "a"}, + []string{"a"}, + }, + { + []string{"a", "a", "a", "a"}, + []string{"a"}, + }, + { + nil, + nil, + }, + { + []string{" a ", " a ", "b ", " b"}, + []string{"a", "b"}, + }, + } + + for _, test := range tests { + result := UniqueTrimmedStrings(test.input) + if len(result) != len(test.expected) { + t.Errorf("%s != %s", result, test.expected) + } + for i := range result { + if test.expected[i] != result[i] { + t.Errorf("%s != %s", result, test.expected) + } + } + } +} diff --git a/lib/util/utils.go b/lib/structutil/structutil.go similarity index 58% rename from lib/util/utils.go rename to lib/structutil/structutil.go index 43fafde78..e60a440f2 100644 --- a/lib/util/utils.go +++ b/lib/structutil/structutil.go @@ -1,19 +1,15 @@ -// Copyright (C) 2016 The Syncthing Authors. +// Copyright (C) 2023 The Syncthing Authors. // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -package util +package structutil import ( - "context" - "fmt" - "net/url" "reflect" "strconv" "strings" - "time" ) type defaultParser interface { @@ -21,7 +17,7 @@ type defaultParser interface { } // SetDefaults sets default values on a struct, based on the default annotation. -func SetDefaults(data interface{}) { +func SetDefaults(data any) { s := reflect.ValueOf(data).Elem() t := s.Type() @@ -86,63 +82,15 @@ func SetDefaults(data interface{}) { } } -// CopyMatchingTag copies fields tagged tag:"value" from "from" struct onto "to" struct. -func CopyMatchingTag(from interface{}, to interface{}, tag string, shouldCopy func(value string) bool) { - fromStruct := reflect.ValueOf(from).Elem() - fromType := fromStruct.Type() - - toStruct := reflect.ValueOf(to).Elem() - toType := toStruct.Type() - - if fromType != toType { - panic(fmt.Sprintf("non equal types: %s != %s", fromType, toType)) - } - - for i := 0; i < toStruct.NumField(); i++ { - fromField := fromStruct.Field(i) - toField := toStruct.Field(i) - - if !toField.CanSet() { - // Unexported fields - continue - } - - structTag := toType.Field(i).Tag - - v := structTag.Get(tag) - if shouldCopy(v) { - toField.Set(fromField) - } - } -} - -// UniqueTrimmedStrings returns a list of all unique strings in ss, -// in the order in which they first appear in ss, after trimming away -// leading and trailing spaces. -func UniqueTrimmedStrings(ss []string) []string { - var m = make(map[string]struct{}, len(ss)) - var us = make([]string, 0, len(ss)) - for _, v := range ss { - v = strings.Trim(v, " ") - if _, ok := m[v]; ok { - continue - } - m[v] = struct{}{} - us = append(us, v) - } - - return us -} - -func FillNilExceptDeprecated(data interface{}) { +func FillNilExceptDeprecated(data any) { fillNil(data, true) } -func FillNil(data interface{}) { +func FillNil(data any) { fillNil(data, false) } -func fillNil(data interface{}, skipDeprecated bool) { +func fillNil(data any, skipDeprecated bool) { s := reflect.ValueOf(data).Elem() t := s.Type() for i := 0; i < s.NumField(); i++ { @@ -190,7 +138,7 @@ func fillNil(data interface{}, skipDeprecated bool) { } // FillNilSlices sets default value on slices that are still nil. -func FillNilSlices(data interface{}) error { +func FillNilSlices(data any) error { s := reflect.ValueOf(data).Elem() t := s.Type() @@ -220,55 +168,3 @@ func FillNilSlices(data interface{}) error { } return nil } - -// Address constructs a URL from the given network and hostname. -func Address(network, host string) string { - u := url.URL{ - Scheme: network, - Host: host, - } - return u.String() -} - -func CallWithContext(ctx context.Context, fn func() error) error { - var err error - done := make(chan struct{}) - go func() { - err = fn() - close(done) - }() - select { - case <-done: - return err - case <-ctx.Done(): - return ctx.Err() - } -} - -func NiceDurationString(d time.Duration) string { - switch { - case d > 24*time.Hour: - d = d.Round(time.Hour) - case d > time.Hour: - d = d.Round(time.Minute) - case d > time.Minute: - d = d.Round(time.Second) - case d > time.Second: - d = d.Round(time.Millisecond) - case d > time.Millisecond: - d = d.Round(time.Microsecond) - } - return d.String() -} - -func EqualStrings(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} diff --git a/lib/util/utils_test.go b/lib/structutil/structutil_test.go similarity index 64% rename from lib/util/utils_test.go rename to lib/structutil/structutil_test.go index d16ef2f48..0ded3b40c 100644 --- a/lib/util/utils_test.go +++ b/lib/structutil/structutil_test.go @@ -4,7 +4,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -package util +package structutil import ( "testing" @@ -55,46 +55,6 @@ func TestSetDefaults(t *testing.T) { } } -func TestUniqueStrings(t *testing.T) { - tests := []struct { - input []string - expected []string - }{ - { - []string{"a", "b"}, - []string{"a", "b"}, - }, - { - []string{"a", "a"}, - []string{"a"}, - }, - { - []string{"a", "a", "a", "a"}, - []string{"a"}, - }, - { - nil, - nil, - }, - { - []string{" a ", " a ", "b ", " b"}, - []string{"a", "b"}, - }, - } - - for _, test := range tests { - result := UniqueTrimmedStrings(test.input) - if len(result) != len(test.expected) { - t.Errorf("%s != %s", result, test.expected) - } - for i := range result { - if test.expected[i] != result[i] { - t.Errorf("%s != %s", result, test.expected) - } - } - } -} - func TestFillNillSlices(t *testing.T) { // Nil x := &struct { @@ -148,83 +108,6 @@ func TestFillNillSlices(t *testing.T) { } } -func TestAddress(t *testing.T) { - tests := []struct { - network string - host string - result string - }{ - {"tcp", "google.com", "tcp://google.com"}, - {"foo", "google", "foo://google"}, - {"123", "456", "123://456"}, - } - - for _, test := range tests { - result := Address(test.network, test.host) - if result != test.result { - t.Errorf("%s != %s", result, test.result) - } - } -} - -func TestCopyMatching(t *testing.T) { - type Nested struct { - A int - } - type Test struct { - CopyA int - CopyB []string - CopyC Nested - CopyD *Nested - NoCopy int `restart:"true"` - } - - from := Test{ - CopyA: 1, - CopyB: []string{"friend", "foe"}, - CopyC: Nested{ - A: 2, - }, - CopyD: &Nested{ - A: 3, - }, - NoCopy: 4, - } - - to := Test{ - CopyA: 11, - CopyB: []string{"foot", "toe"}, - CopyC: Nested{ - A: 22, - }, - CopyD: &Nested{ - A: 33, - }, - NoCopy: 44, - } - - // Copy empty fields - CopyMatchingTag(&from, &to, "restart", func(v string) bool { - return v != "true" - }) - - if to.CopyA != 1 { - t.Error("CopyA") - } - if len(to.CopyB) != 2 || to.CopyB[0] != "friend" || to.CopyB[1] != "foe" { - t.Error("CopyB") - } - if to.CopyC.A != 2 { - t.Error("CopyC") - } - if to.CopyD.A != 3 { - t.Error("CopyC") - } - if to.NoCopy != 44 { - t.Error("NoCopy") - } -} - func TestFillNil(t *testing.T) { type A struct { Slice []int diff --git a/lib/stun/stun.go b/lib/stun/stun.go index 191b39c5b..6f776d972 100644 --- a/lib/stun/stun.go +++ b/lib/stun/stun.go @@ -14,7 +14,7 @@ import ( "github.com/ccding/go-stun/stun" "github.com/syncthing/syncthing/lib/config" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/svcutil" ) const stunRetryInterval = 5 * time.Minute @@ -159,7 +159,7 @@ func (s *Service) runStunForServer(ctx context.Context, addr string) { var natType stun.NATType var extAddr *stun.Host - err = util.CallWithContext(ctx, func() error { + err = svcutil.CallWithContext(ctx, func() error { natType, extAddr, err = s.client.Discover() return err }) diff --git a/lib/svcutil/svcutil.go b/lib/svcutil/svcutil.go index f9d5684b0..1db3d800a 100644 --- a/lib/svcutil/svcutil.go +++ b/lib/svcutil/svcutil.go @@ -223,3 +223,18 @@ func asNonContextError(ctx context.Context, err error) error { } return err } + +func CallWithContext(ctx context.Context, fn func() error) error { + var err error + done := make(chan struct{}) + go func() { + err = fn() + close(done) + }() + select { + case <-done: + return err + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/lib/testutils/testutils.go b/lib/testutil/testutil.go similarity index 98% rename from lib/testutils/testutils.go rename to lib/testutil/testutil.go index 09dc7e805..523b3a8d5 100644 --- a/lib/testutils/testutils.go +++ b/lib/testutil/testutil.go @@ -4,7 +4,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -package testutils +package testutil import ( "errors" @@ -25,6 +25,7 @@ func NewBlockingRW() *BlockingRW { closeOnce: sync.Once{}, } } + func (rw *BlockingRW) Read(_ []byte) (int, error) { <-rw.c return 0, ErrClosed diff --git a/lib/ur/contract/contract.go b/lib/ur/contract/contract.go index daae595cb..52f07ab2f 100644 --- a/lib/ur/contract/contract.go +++ b/lib/ur/contract/contract.go @@ -14,7 +14,7 @@ import ( "strconv" "time" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/structutil" ) type Report struct { @@ -179,7 +179,7 @@ type Report struct { func New() *Report { r := &Report{} - util.FillNil(r) + structutil.FillNil(r) return r } diff --git a/lib/versioner/util.go b/lib/versioner/util.go index ffa0dd3fe..d01f33c0f 100644 --- a/lib/versioner/util.go +++ b/lib/versioner/util.go @@ -19,7 +19,7 @@ import ( "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/osutil" - "github.com/syncthing/syncthing/lib/util" + "github.com/syncthing/syncthing/lib/stringutil" ) var ( @@ -126,7 +126,6 @@ func retrieveVersions(fileSystem fs.Filesystem) (map[string][]FileVersion, error return nil }) - if err != nil { return nil, err } @@ -153,7 +152,7 @@ func archiveFile(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, filePath if err != nil { if fs.IsNotExist(err) { l.Debugln("creating versions dir") - err := dstFs.MkdirAll(".", 0755) + err := dstFs.MkdirAll(".", 0o755) if err != nil { return err } @@ -166,7 +165,7 @@ func archiveFile(method fs.CopyRangeMethod, srcFs, dstFs fs.Filesystem, filePath file := filepath.Base(filePath) inFolderPath := filepath.Dir(filePath) - err = dstFs.MkdirAll(inFolderPath, 0755) + err = dstFs.MkdirAll(inFolderPath, 0o755) if err != nil && !fs.IsExist(err) { l.Debugln("archiving", filePath, err) return err @@ -253,7 +252,7 @@ func restoreFile(method fs.CopyRangeMethod, src, dst fs.Filesystem, filePath str return err } - _ = dst.MkdirAll(filepath.Dir(filePath), 0755) + _ = dst.MkdirAll(filepath.Dir(filePath), 0o755) err := osutil.RenameOrCopy(method, src, dst, sourceFile, filePath) _ = dst.Chtimes(filePath, sourceMtime, sourceMtime) return err @@ -285,7 +284,7 @@ func findAllVersions(fs fs.Filesystem, filePath string) []string { l.Warnln("globbing:", err, "for", pattern) return nil } - versions = util.UniqueTrimmedStrings(versions) + versions = stringutil.UniqueTrimmedStrings(versions) sort.Strings(versions) return versions