Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move index to index-v0.11.0.db (new format) and centralize location config

This commit is contained in:
Jakob Borg 2015-03-29 12:55:27 +02:00
parent a3cf37cb2e
commit 54752deaa1
20 changed files with 184 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/syncthing/gui.go
View File

@ -58,7 +58,7 @@ func init() {
func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) error {
var err error
cert, err := loadCert(confDir, "https-")
cert, err := tls.LoadX509KeyPair(locations[locHttpsCertFile], locations[locHttpsKeyFile])
if err != nil {
l.Infoln("Loading HTTPS certificate:", err)
l.Infoln("Creating new HTTPS certificate")
@ -71,8 +71,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
name = tlsDefaultCommonName
}
newCertificate(confDir, "https-", name)
cert, err = loadCert(confDir, "https-")
cert, err = newCertificate(locations[locHttpsCertFile], locations[locHttpsKeyFile], name)
}
if err != nil {
return err

6
cmd/syncthing/gui_csrf.go
View File

@ -11,7 +11,6 @@ import (
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
@ -92,7 +91,7 @@ func newCsrfToken() string {
}
func saveCsrfTokens() {
name := filepath.Join(confDir, "csrftokens.txt")
name := locations[locCsrfTokens]
tmp := fmt.Sprintf("%s.tmp.%d", name, time.Now().UnixNano())
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
@ -117,8 +116,7 @@ func saveCsrfTokens() {
}
func loadCsrfTokens() {
name := filepath.Join(confDir, "csrftokens.txt")
f, err := os.Open(name)
f, err := os.Open(locations[locCsrfTokens])
if err != nil {
return
}

109
cmd/syncthing/locations.go Normal file
View File

@ -0,0 +1,109 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
package main
import (
"os"
"path/filepath"
"runtime"
"strings"
"github.com/syncthing/syncthing/internal/osutil"
)
type locationEnum string
// Use strings as keys to make printout and serialization of the locations map
// more meaningful.
const (
locConfigFile locationEnum = "config"
locCertFile = "certFile"
locKeyFile = "keyFile"
locHttpsCertFile = "httpsCertFile"
locHttpsKeyFile = "httpsKeyFile"
locDatabase = "database"
locLogFile = "logFile"
locCsrfTokens = "csrfTokens"
locPanicLog = "panicLog"
locDefFolder = "defFolder"
)
// Platform dependent directories
var baseDirs = map[string]string{
"config": defaultConfigDir(), // Overridden by -home flag
"home": homeDir(), // User's home directory, *not* -home flag
}
// Use the variables from baseDirs here
var locations = map[locationEnum]string{
locConfigFile: "${config}/config.xml",
locCertFile: "${config}/cert.pem",
locKeyFile: "${config}/key.pem",
locHttpsCertFile: "${config}/https-cert.pem",
locHttpsKeyFile: "${config}/https-key.pem",
locDatabase: "${config}/index-v0.11.0.db",
locLogFile: "${config}/syncthing.log", // -logfile on Windows
locCsrfTokens: "${config}/csrftokens.txt",
locPanicLog: "${config}/panic-20060102-150405.log", // passed through time.Format()
locDefFolder: "${home}/Sync",
}
// expandLocations replaces the variables in the location map with actual
// directory locations.
func expandLocations() error {
for key, dir := range locations {
for varName, value := range baseDirs {
dir = strings.Replace(dir, "${"+varName+"}", value, -1)
}
var err error
dir, err = osutil.ExpandTilde(dir)
if err != nil {
return err
}
locations[key] = dir
}
return nil
}
// defaultConfigDir returns the default configuration directory, as figured
// out by various the environment variables present on each platform, or dies
// trying.
func defaultConfigDir() string {
switch runtime.GOOS {
case "windows":
if p := os.Getenv("LocalAppData"); p != "" {
return filepath.Join(p, "Syncthing")
}
return filepath.Join(os.Getenv("AppData"), "Syncthing")
case "darwin":
dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
if err != nil {
l.Fatalln(err)
}
return dir
default:
if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
return filepath.Join(xdgCfg, "syncthing")
}
dir, err := osutil.ExpandTilde("~/.config/syncthing")
if err != nil {
l.Fatalln(err)
}
return dir
}
}
// homeDir returns the user's home directory, or dies trying.
func homeDir() string {
home, err := osutil.ExpandTilde("~")
if err != nil {
l.Fatalln(err)
}
return home
}

99
cmd/syncthing/main.go
View File

@ -196,17 +196,11 @@ var (
)
func main() {
defConfDir, err := getDefaultConfDir()
if err != nil {
l.Fatalln("home:", err)
}
if runtime.GOOS == "windows" {
// On Windows, we use a log file by default. Setting the -logfile flag
// to the empty string disables this behavior.
// to "-" disables this behavior.
logFile = filepath.Join(defConfDir, "syncthing.log")
flag.StringVar(&logFile, "logfile", logFile, "Log file name (blank for stdout)")
flag.StringVar(&logFile, "logfile", "", "Log file name (use \"-\" for stdout)")
// We also add an option to hide the console window
flag.BoolVar(&noConsole, "no-console", false, "Hide console window")
@ -226,23 +220,30 @@ func main() {
flag.BoolVar(&showVersion, "version", false, "Show version")
flag.StringVar(&upgradeTo, "upgrade-to", upgradeTo, "Force upgrade directly from specified URL")
flag.Usage = usageFor(flag.CommandLine, usage, fmt.Sprintf(extraUsage, defConfDir))
flag.Usage = usageFor(flag.CommandLine, usage, fmt.Sprintf(extraUsage, baseDirs["config"]))
flag.Parse()
if noConsole {
osutil.HideConsole()
}
if confDir == "" {
if confDir != "" {
// Not set as default above because the string can be really long.
confDir = defConfDir
baseDirs["config"] = confDir
}
if confDir != defConfDir && filepath.Dir(logFile) == defConfDir {
// The user changed the config dir with -home, but not the log file
// location. In this case we assume they meant for the logfile to
// still live in it's default location *relative to the config dir*.
logFile = filepath.Join(confDir, "syncthing.log")
if runtime.GOOS == "windows" {
if logFile == "" {
// Use the default log file location
logFile = locations[locLogFile]
} else if logFile == "-" {
// Don't use a logFile
logFile = ""
}
}
if err := expandLocations(); err != nil {
l.Fatalln(err)
}
if showVersion {
@ -269,13 +270,13 @@ func main() {
}
}
cert, err := loadCert(dir, "")
certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err == nil {
l.Warnln("Key exists; will not overwrite.")
l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
} else {
newCertificate(dir, "", tlsDefaultCommonName)
cert, err = loadCert(dir, "")
cert, err = newCertificate(certFile, keyFile, tlsDefaultCommonName)
myID = protocol.NewDeviceID(cert.Certificate[0])
if err != nil {
l.Fatalln("load cert:", err)
@ -301,17 +302,12 @@ func main() {
return
}
confDir, err := osutil.ExpandTilde(confDir)
if err != nil {
l.Fatalln("home:", err)
}
if info, err := os.Stat(confDir); err == nil && !info.IsDir() {
l.Fatalln("Config directory", confDir, "is not a directory")
if info, err := os.Stat(baseDirs["config"]); err == nil && !info.IsDir() {
l.Fatalln("Config directory", baseDirs["config"], "is not a directory")
}
// Ensure that our home directory exists.
ensureDir(confDir, 0700)
ensureDir(baseDirs["config"], 0700)
if upgradeTo != "" {
err := upgrade.ToURL(upgradeTo)
@ -337,7 +333,7 @@ func main() {
if doUpgrade {
// Use leveldb database locks to protect against concurrent upgrades
_, err = leveldb.OpenFile(filepath.Join(confDir, "index"), &opt.Options{OpenFilesCacheCapacity: 100})
_, err = leveldb.OpenFile(locations[locDatabase], &opt.Options{OpenFilesCacheCapacity: 100})
if err != nil {
l.Fatalln("Cannot upgrade, database seems to be locked. Is another copy of Syncthing already running?")
}
@ -371,13 +367,12 @@ func syncthingMain() {
runtime.GOMAXPROCS(runtime.NumCPU())
}
events.Default.Log(events.Starting, map[string]string{"home": confDir})
events.Default.Log(events.Starting, map[string]string{"home": baseDirs["config"]})
// Ensure that that we have a certificate and key.
cert, err = loadCert(confDir, "")
cert, err = tls.LoadX509KeyPair(locations[locCertFile], locations[locKeyFile])
if err != nil {
newCertificate(confDir, "", tlsDefaultCommonName)
cert, err = loadCert(confDir, "")
cert, err = newCertificate(locations[locCertFile], locations[locKeyFile], tlsDefaultCommonName)
if err != nil {
l.Fatalln("load cert:", err)
}
@ -395,7 +390,7 @@ func syncthingMain() {
// Prepare to be able to save configuration
cfgFile := filepath.Join(confDir, "config.xml")
cfgFile := locations[locConfigFile]
var myName string
@ -490,7 +485,7 @@ func syncthingMain() {
l.Infoln("Local networks:", strings.Join(networks, ", "))
}
dbFile := filepath.Join(confDir, "index")
dbFile := locations[locDatabase]
dbOpts := &opt.Options{OpenFilesCacheCapacity: 100}
ldb, err := leveldb.OpenFile(dbFile, dbOpts)
if err != nil && errors.IsCorrupted(err) {
@ -665,16 +660,11 @@ func setupGUI(cfg *config.Wrapper, m *model.Model) {
}
func defaultConfig(myName string) config.Configuration {
defaultFolder, err := osutil.ExpandTilde("~/Sync")
if err != nil {
l.Fatalln("home:", err)
}
newCfg := config.New(myID)
newCfg.Folders = []config.FolderConfiguration{
{
ID: "default",
Path: defaultFolder,
Path: locations[locDefFolder],
RescanIntervalS: 60,
Devices: []config.FolderDeviceConfiguration{{DeviceID: myID}},
},
@ -811,12 +801,7 @@ func renewUPnP(port int) {
}
func resetFolders() {
confDir, err := osutil.ExpandTilde(confDir)
if err != nil {
log.Fatal(err)
}
cfgFile := filepath.Join(confDir, "config.xml")
cfgFile := locations[locConfigFile]
cfg, err := config.Load(cfgFile, myID)
if err != nil {
log.Fatal(err)
@ -832,8 +817,7 @@ func resetFolders() {
}
}
idx := filepath.Join(confDir, "index")
os.RemoveAll(idx)
os.RemoveAll(locations[locDatabase])
}
func restart() {
@ -879,25 +863,6 @@ func ensureDir(dir string, mode int) {
}
}
func getDefaultConfDir() (string, error) {
switch runtime.GOOS {
case "windows":
if p := os.Getenv("LocalAppData"); p != "" {
return filepath.Join(p, "Syncthing"), nil
}
return filepath.Join(os.Getenv("AppData"), "Syncthing"), nil
case "darwin":
return osutil.ExpandTilde("~/Library/Application Support/Syncthing")
default:
if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
return filepath.Join(xdgCfg, "syncthing"), nil
}
return osutil.ExpandTilde("~/.config/syncthing")
}
}
// getFreePort returns a free TCP port fort listening on. The ports given are
// tried in succession and the first to succeed is returned. If none succeed,
// a random high port is returned.

3
cmd/syncthing/monitor.go
View File

@ -12,7 +12,6 @@ import (
"os"
"os/exec"
"os/signal"
"path/filepath"
"runtime"
"strings"
"sync"
@ -164,7 +163,7 @@ func copyStderr(stderr io.ReadCloser, dst io.Writer) {
dst.Write([]byte(line))
if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
panicFd, err = os.Create(filepath.Join(confDir, time.Now().Format("panic-20060102-150405.log")))
panicFd, err = os.Create(time.Now().Format(locations[locPanicLog]))
if err != nil {
l.Warnln("Create panic log:", err)
continue

15
cmd/syncthing/tls.go
View File

@ -19,7 +19,6 @@ import (
mr "math/rand"
"net"
"os"
"path/filepath"
"time"
)
@ -28,13 +27,7 @@ const (
tlsDefaultCommonName = "syncthing"
)
func loadCert(dir string, prefix string) (tls.Certificate, error) {
cf := filepath.Join(dir, prefix+"cert.pem")
kf := filepath.Join(dir, prefix+"key.pem")
return tls.LoadX509KeyPair(cf, kf)
}
func newCertificate(dir, prefix, name string) {
func newCertificate(certFile, keyFile, name string) (tls.Certificate, error) {
l.Infof("Generating RSA key and certificate for %s...", name)
priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
@ -63,7 +56,7 @@ func newCertificate(dir, prefix, name string) {
l.Fatalln("create cert:", err)
}
certOut, err := os.Create(filepath.Join(dir, prefix+"cert.pem"))
certOut, err := os.Create(certFile)
if err != nil {
l.Fatalln("save cert:", err)
}
@ -76,7 +69,7 @@ func newCertificate(dir, prefix, name string) {
l.Fatalln("save cert:", err)
}
keyOut, err := os.OpenFile(filepath.Join(dir, prefix+"key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
keyOut, err := os.OpenFile(keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
l.Fatalln("save key:", err)
}
@ -88,6 +81,8 @@ func newCertificate(dir, prefix, name string) {
if err != nil {
l.Fatalln("save key:", err)
}
return tls.LoadX509KeyPair(certFile, keyFile)
}
type DowngradingListener struct {

2
test/.gitignore vendored
View File

@ -15,5 +15,5 @@ dirs-*
csrftokens.txt
s4d
http
index
h*/index*
*.syncthing-reset*

2
test/cli_test.go
View File

@ -17,7 +17,7 @@ import (
)
func TestCLIReset(t *testing.T) {
dirs := []string{"s1", "s12-1", "h1/index"}
dirs := []string{"s1", "s12-1", "h1/index-v0.11.0.db"}
// Create directories that reset will remove

9
test/conflict_test.go
View File

@ -18,7 +18,7 @@ import (
func TestConflict(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index", "h2/index")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}
@ -64,16 +64,11 @@ func TestConflict(t *testing.T) {
// startup, UPnP etc complete and make sure the sender has the full index
// before they connect.
for i := 0; i < 20; i++ {
resp, err := sender.post("/rest/scan?folder=default", nil)
err := sender.rescan("default")
if err != nil {
time.Sleep(time.Second)
continue
}
if resp.StatusCode != 200 {
resp.Body.Close()
time.Sleep(time.Second)
continue
}
break
}

2
test/filetype_test.go
View File

@ -61,7 +61,7 @@ func TestFileTypeChangeStaggeredVersioning(t *testing.T) {
func testFileTypeChange(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index", "h2/index")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}

4
test/h2/config.xml
View File

@ -3,7 +3,9 @@
<device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
<device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
<device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
<versioning type="staggered"></versioning>
<versioning type="simple">
<param key="keep" val="5"></param>
</versioning>
<lenientMtimes>false</lenientMtimes>
<copiers>1</copiers>
<pullers>16</pullers>

2
test/httpstress_test.go
View File

@ -23,7 +23,7 @@ import (
func TestStressHTTP(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s2", "h2/index")
err := removeAll("s2", "h2/index*")
if err != nil {
t.Fatal(err)
}

2
test/ignore_test.go
View File

@ -22,7 +22,7 @@ func TestIgnores(t *testing.T) {
// Clean and start a syncthing instance
log.Println("Cleaning...")
err := removeAll("s1", "h1/index")
err := removeAll("s1", "h1/index*")
if err != nil {
t.Fatal(err)
}

2
test/manypeers_test.go
View File

@ -22,7 +22,7 @@ import (
func TestManyPeers(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index", "h2/index")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}

2
test/parallell_scan_test.go
View File

@ -18,7 +18,7 @@ import (
func TestParallellScan(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "h1/index")
err := removeAll("s1", "h1/index*")
if err != nil {
t.Fatal(err)
}

2
test/reconnect_test.go
View File

@ -32,7 +32,7 @@ func TestRestartSenderAndReceiverDuringTransfer(t *testing.T) {
func testRestartDuringTransfer(t *testing.T, restartSender, restartReceiver bool, senderDelay, receiverDelay time.Duration) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index", "h2/index")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}

2
test/symlink_test.go
View File

@ -86,7 +86,7 @@ func TestSymlinksStaggeredVersioning(t *testing.T) {
func testSymlinks(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index", "h2/index")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}

2
test/sync_test.go
View File

@ -79,7 +79,7 @@ func testSyncCluster(t *testing.T) {
err := removeAll("s1", "s12-1",
"s2", "s12-2", "s23-2",
"s3", "s23-3",
"h1/index", "h2/index", "h3/index")
"h1/index*", "h2/index*", "h3/index*")
if err != nil {
t.Fatal(err)
}

2
test/transfer-bench_test.go
View File

@ -16,7 +16,7 @@ import (
func TestBenchmarkTransfer(t *testing.T) {
log.Println("Cleaning...")
err := removeAll("s1", "s2", "h1/index", "h2/index")
err := removeAll("s1", "s2", "h1/index*", "h2/index*")
if err != nil {
t.Fatal(err)
}

28
test/util.go
View File

@ -241,17 +241,23 @@ func (i *inifiteReader) Read(bs []byte) (int, error) {
// rm -rf
func removeAll(dirs ...string) error {
for _, dir := range dirs {
// Set any non-writeable files and dirs to writeable. This is necessary for os.RemoveAll to work on Windows.
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode()&0700 != 0700 {
os.Chmod(path, 0777)
}
return nil
})
os.RemoveAll(dir)
files, err := filepath.Glob(dir)
if err != nil {
return err
}
for _, file := range files {
// Set any non-writeable files and dirs to writeable. This is necessary for os.RemoveAll to work on Windows.
filepath.Walk(file, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode()&0700 != 0700 {
os.Chmod(path, 0777)
}
return nil
})
os.RemoveAll(file)
}
}
return nil
}