2014-02-12 23:18:41 +01:00
|
|
|
package main
|
2013-12-15 11:43:31 +01:00
|
|
|
|
|
|
|
import (
|
2014-03-29 18:53:48 +01:00
|
|
|
"compress/gzip"
|
2014-01-06 11:11:18 +01:00
|
|
|
"crypto/sha1"
|
2014-01-06 21:31:36 +01:00
|
|
|
"errors"
|
2013-12-23 18:12:44 +01:00
|
|
|
"fmt"
|
2014-01-01 03:22:49 +01:00
|
|
|
"io"
|
2014-01-05 23:54:57 +01:00
|
|
|
"net"
|
2013-12-15 11:43:31 +01:00
|
|
|
"os"
|
2014-03-28 14:36:57 +01:00
|
|
|
"path/filepath"
|
2013-12-15 11:43:31 +01:00
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/calmh/syncthing/buffers"
|
2014-03-28 14:36:57 +01:00
|
|
|
"github.com/calmh/syncthing/cid"
|
|
|
|
"github.com/calmh/syncthing/files"
|
|
|
|
"github.com/calmh/syncthing/lamport"
|
2013-12-15 11:43:31 +01:00
|
|
|
"github.com/calmh/syncthing/protocol"
|
2014-03-08 23:02:01 +01:00
|
|
|
"github.com/calmh/syncthing/scanner"
|
2013-12-15 11:43:31 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
type Model struct {
|
2014-03-29 18:53:48 +01:00
|
|
|
repoDirs map[string]string // repo -> dir
|
|
|
|
repoFiles map[string]*files.Set // repo -> files
|
|
|
|
repoNodes map[string][]string // repo -> nodeIDs
|
|
|
|
nodeRepos map[string][]string // nodeID -> repos
|
|
|
|
rmut sync.RWMutex // protects the above
|
|
|
|
|
|
|
|
cm *cid.Map
|
2013-12-30 15:30:29 +01:00
|
|
|
|
2014-03-28 14:36:57 +01:00
|
|
|
protoConn map[string]protocol.Connection
|
2014-01-09 13:58:35 +01:00
|
|
|
rawConn map[string]io.Closer
|
2014-01-18 04:06:44 +01:00
|
|
|
pmut sync.RWMutex // protects protoConn and rawConn
|
2013-12-30 15:30:29 +01:00
|
|
|
|
2014-01-22 11:38:52 +01:00
|
|
|
sup suppressor
|
2014-01-12 16:59:35 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
addedRepo bool
|
|
|
|
started bool
|
2013-12-15 11:43:31 +01:00
|
|
|
}
|
|
|
|
|
2014-01-07 22:44:21 +01:00
|
|
|
var (
|
|
|
|
ErrNoSuchFile = errors.New("no such file")
|
|
|
|
ErrInvalid = errors.New("file is invalid")
|
|
|
|
)
|
2014-01-06 21:31:36 +01:00
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
// NewModel creates and starts a new model. The model starts in read-only mode,
|
|
|
|
// where it sends index information to connected peers and responds to requests
|
|
|
|
// for file data without altering the local repository in any way.
|
2014-03-29 18:53:48 +01:00
|
|
|
func NewModel(maxChangeBw int) *Model {
|
2013-12-15 11:43:31 +01:00
|
|
|
m := &Model{
|
2014-03-29 18:53:48 +01:00
|
|
|
repoDirs: make(map[string]string),
|
|
|
|
repoFiles: make(map[string]*files.Set),
|
|
|
|
repoNodes: make(map[string][]string),
|
|
|
|
nodeRepos: make(map[string][]string),
|
2014-03-28 14:36:57 +01:00
|
|
|
cm: cid.NewMap(),
|
|
|
|
protoConn: make(map[string]protocol.Connection),
|
|
|
|
rawConn: make(map[string]io.Closer),
|
|
|
|
sup: suppressor{threshold: int64(maxChangeBw)},
|
2013-12-15 11:43:31 +01:00
|
|
|
}
|
|
|
|
|
2013-12-24 21:21:03 +01:00
|
|
|
go m.broadcastIndexLoop()
|
2013-12-15 11:43:31 +01:00
|
|
|
return m
|
|
|
|
}
|
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
// StartRW starts read/write processing on the current model. When in
|
|
|
|
// read/write mode the model will attempt to keep in sync with the cluster by
|
|
|
|
// pulling needed files from peer nodes.
|
2014-04-08 13:45:18 +02:00
|
|
|
func (m *Model) StartRepoRW(repo string, threads int) {
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.Lock()
|
|
|
|
defer m.rmut.Unlock()
|
|
|
|
|
2014-04-08 13:45:18 +02:00
|
|
|
if dir, ok := m.repoDirs[repo]; !ok {
|
2014-03-29 18:53:48 +01:00
|
|
|
panic("cannot start without repo")
|
2014-04-08 13:45:18 +02:00
|
|
|
} else {
|
2014-03-29 18:53:48 +01:00
|
|
|
newPuller(repo, dir, m, threads)
|
|
|
|
}
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
2014-01-06 11:11:18 +01:00
|
|
|
|
2014-03-28 14:36:57 +01:00
|
|
|
// StartRO starts read only processing on the current model. When in
|
|
|
|
// read only mode the model will announce files to the cluster but not
|
|
|
|
// pull in any external changes.
|
2014-04-08 13:45:18 +02:00
|
|
|
func (m *Model) StartRepoRO(repo string) {
|
|
|
|
m.StartRepoRW(repo, 0) // zero threads => read only
|
2014-01-20 22:22:27 +01:00
|
|
|
}
|
|
|
|
|
2014-01-05 23:54:57 +01:00
|
|
|
type ConnectionInfo struct {
|
|
|
|
protocol.Statistics
|
2014-01-23 13:12:45 +01:00
|
|
|
Address string
|
|
|
|
ClientID string
|
|
|
|
ClientVersion string
|
2014-02-13 12:41:37 +01:00
|
|
|
Completion int
|
2014-01-05 23:54:57 +01:00
|
|
|
}
|
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
// ConnectionStats returns a map with connection statistics for each connected node.
|
2014-01-05 23:54:57 +01:00
|
|
|
func (m *Model) ConnectionStats() map[string]ConnectionInfo {
|
|
|
|
type remoteAddrer interface {
|
|
|
|
RemoteAddr() net.Addr
|
|
|
|
}
|
|
|
|
|
2014-01-18 04:06:44 +01:00
|
|
|
m.pmut.RLock()
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RLock()
|
2014-01-05 16:16:37 +01:00
|
|
|
|
2014-01-05 23:54:57 +01:00
|
|
|
var res = make(map[string]ConnectionInfo)
|
2014-01-09 13:58:35 +01:00
|
|
|
for node, conn := range m.protoConn {
|
2014-01-05 23:54:57 +01:00
|
|
|
ci := ConnectionInfo{
|
2014-01-23 13:12:45 +01:00
|
|
|
Statistics: conn.Statistics(),
|
|
|
|
ClientID: conn.Option("clientId"),
|
|
|
|
ClientVersion: conn.Option("clientVersion"),
|
2014-01-05 23:54:57 +01:00
|
|
|
}
|
|
|
|
if nc, ok := m.rawConn[node].(remoteAddrer); ok {
|
|
|
|
ci.Address = nc.RemoteAddr().String()
|
|
|
|
}
|
2014-02-13 12:41:37 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
var tot int64
|
|
|
|
var have int64
|
|
|
|
|
|
|
|
for _, repo := range m.nodeRepos[node] {
|
|
|
|
for _, f := range m.repoFiles[repo].Global() {
|
|
|
|
if f.Flags&protocol.FlagDeleted == 0 {
|
|
|
|
tot += f.Size
|
|
|
|
have += f.Size
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, f := range m.repoFiles[repo].Need(m.cm.Get(node)) {
|
|
|
|
if f.Flags&protocol.FlagDeleted == 0 {
|
|
|
|
have -= f.Size
|
|
|
|
}
|
2014-02-13 12:41:37 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-03-03 08:47:52 +01:00
|
|
|
ci.Completion = 100
|
|
|
|
if tot != 0 {
|
|
|
|
ci.Completion = int(100 * have / tot)
|
|
|
|
}
|
2014-02-13 12:41:37 +01:00
|
|
|
|
2014-01-05 23:54:57 +01:00
|
|
|
res[node] = ci
|
2013-12-30 15:30:29 +01:00
|
|
|
}
|
2014-01-18 04:06:44 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RUnlock()
|
2014-01-18 04:06:44 +01:00
|
|
|
m.pmut.RUnlock()
|
2014-03-28 14:36:57 +01:00
|
|
|
|
2014-01-05 16:16:37 +01:00
|
|
|
return res
|
2013-12-30 15:30:29 +01:00
|
|
|
}
|
|
|
|
|
2014-03-28 14:36:57 +01:00
|
|
|
func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
|
|
|
|
for _, f := range fs {
|
2014-01-07 22:44:21 +01:00
|
|
|
if f.Flags&protocol.FlagDeleted == 0 {
|
2014-01-05 23:54:57 +01:00
|
|
|
files++
|
2014-02-20 17:40:15 +01:00
|
|
|
bytes += f.Size
|
2014-01-05 23:54:57 +01:00
|
|
|
} else {
|
|
|
|
deleted++
|
|
|
|
}
|
2013-12-30 15:30:29 +01:00
|
|
|
}
|
2014-01-05 16:16:37 +01:00
|
|
|
return
|
|
|
|
}
|
2013-12-30 15:30:29 +01:00
|
|
|
|
2014-03-28 14:36:57 +01:00
|
|
|
// GlobalSize returns the number of files, deleted files and total bytes for all
|
|
|
|
// files in the global model.
|
|
|
|
func (m *Model) GlobalSize() (files, deleted int, bytes int64) {
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RLock()
|
|
|
|
var fs []scanner.File
|
|
|
|
for _, rf := range m.repoFiles {
|
|
|
|
fs = append(fs, rf.Global()...)
|
|
|
|
}
|
|
|
|
m.rmut.RUnlock()
|
2014-03-28 14:36:57 +01:00
|
|
|
return sizeOf(fs)
|
|
|
|
}
|
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
// LocalSize returns the number of files, deleted files and total bytes for all
|
|
|
|
// files in the local repository.
|
2014-02-20 17:40:15 +01:00
|
|
|
func (m *Model) LocalSize() (files, deleted int, bytes int64) {
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RLock()
|
|
|
|
var fs []scanner.File
|
|
|
|
for _, rf := range m.repoFiles {
|
|
|
|
fs = append(fs, rf.Have(cid.LocalID)...)
|
|
|
|
}
|
|
|
|
m.rmut.RUnlock()
|
2014-03-28 14:36:57 +01:00
|
|
|
return sizeOf(fs)
|
2013-12-30 15:30:29 +01:00
|
|
|
}
|
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
// InSyncSize returns the number and total byte size of the local files that
|
|
|
|
// are in sync with the global model.
|
2014-03-28 14:36:57 +01:00
|
|
|
func (m *Model) InSyncSize() (files int, bytes int64) {
|
2014-03-29 18:53:48 +01:00
|
|
|
var gf []scanner.File
|
|
|
|
var nf []scanner.File
|
|
|
|
|
|
|
|
m.rmut.RLock()
|
|
|
|
for _, rf := range m.repoFiles {
|
|
|
|
gf = append(gf, rf.Global()...)
|
|
|
|
nf = append(nf, rf.Need(cid.LocalID)...)
|
|
|
|
}
|
|
|
|
m.rmut.RUnlock()
|
2014-01-06 06:38:01 +01:00
|
|
|
|
2014-03-28 14:36:57 +01:00
|
|
|
gn, _, gb := sizeOf(gf)
|
2014-03-29 18:53:48 +01:00
|
|
|
nn, _, nb := sizeOf(nf)
|
2014-01-18 04:06:44 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
return gn - nn, gb - nb
|
2014-01-06 06:38:01 +01:00
|
|
|
}
|
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
// NeedFiles returns the list of currently needed files and the total size.
|
2014-03-28 14:36:57 +01:00
|
|
|
func (m *Model) NeedFiles() ([]scanner.File, int64) {
|
2014-03-29 18:53:48 +01:00
|
|
|
var nf []scanner.File
|
|
|
|
m.rmut.RLock()
|
|
|
|
for _, rf := range m.repoFiles {
|
|
|
|
nf = append(nf, rf.Need(cid.LocalID)...)
|
|
|
|
}
|
|
|
|
m.rmut.RUnlock()
|
2014-01-05 16:16:37 +01:00
|
|
|
|
2014-03-28 14:36:57 +01:00
|
|
|
var bytes int64
|
|
|
|
for _, f := range nf {
|
2014-02-20 17:40:15 +01:00
|
|
|
bytes += f.Size
|
2013-12-23 18:12:44 +01:00
|
|
|
}
|
2014-01-18 04:06:44 +01:00
|
|
|
|
2014-03-28 14:36:57 +01:00
|
|
|
return nf, bytes
|
2013-12-23 18:12:44 +01:00
|
|
|
}
|
|
|
|
|
2014-04-01 23:18:32 +02:00
|
|
|
// NeedFiles returns the list of currently needed files and the total size.
|
|
|
|
func (m *Model) NeedFilesRepo(repo string) []scanner.File {
|
|
|
|
m.rmut.RLock()
|
|
|
|
nf := m.repoFiles[repo].Need(cid.LocalID)
|
|
|
|
m.rmut.RUnlock()
|
|
|
|
return nf
|
|
|
|
}
|
|
|
|
|
2013-12-30 15:30:29 +01:00
|
|
|
// Index is called when a new node is connected and we receive their full index.
|
2014-01-06 11:11:18 +01:00
|
|
|
// Implements the protocol.Model interface.
|
2014-03-29 18:53:48 +01:00
|
|
|
func (m *Model) Index(nodeID string, repo string, fs []protocol.FileInfo) {
|
|
|
|
if debugNet {
|
|
|
|
dlog.Printf("IDX(in): %s / %q: %d files", nodeID, repo, len(fs))
|
|
|
|
}
|
|
|
|
|
2014-03-08 23:02:01 +01:00
|
|
|
var files = make([]scanner.File, len(fs))
|
2014-01-23 22:20:15 +01:00
|
|
|
for i := range fs {
|
2014-03-28 14:36:57 +01:00
|
|
|
lamport.Default.Tick(fs[i].Version)
|
2014-01-23 22:20:15 +01:00
|
|
|
files[i] = fileFromFileInfo(fs[i])
|
|
|
|
}
|
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
id := m.cm.Get(nodeID)
|
|
|
|
m.rmut.RLock()
|
|
|
|
if r, ok := m.repoFiles[repo]; ok {
|
|
|
|
r.Replace(id, files)
|
|
|
|
} else {
|
|
|
|
warnf("Index from %s for nonexistant repo %q; dropping", nodeID, repo)
|
2013-12-15 11:43:31 +01:00
|
|
|
}
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RUnlock()
|
2013-12-28 14:10:36 +01:00
|
|
|
}
|
|
|
|
|
2013-12-30 15:30:29 +01:00
|
|
|
// IndexUpdate is called for incremental updates to connected nodes' indexes.
|
2014-01-06 11:11:18 +01:00
|
|
|
// Implements the protocol.Model interface.
|
2014-03-29 18:53:48 +01:00
|
|
|
func (m *Model) IndexUpdate(nodeID string, repo string, fs []protocol.FileInfo) {
|
|
|
|
if debugNet {
|
|
|
|
dlog.Printf("IDXUP(in): %s / %q: %d files", nodeID, repo, len(fs))
|
|
|
|
}
|
|
|
|
|
2014-03-08 23:02:01 +01:00
|
|
|
var files = make([]scanner.File, len(fs))
|
2014-01-23 22:20:15 +01:00
|
|
|
for i := range fs {
|
2014-03-28 14:36:57 +01:00
|
|
|
lamport.Default.Tick(fs[i].Version)
|
2014-01-23 22:20:15 +01:00
|
|
|
files[i] = fileFromFileInfo(fs[i])
|
|
|
|
}
|
|
|
|
|
2014-03-28 14:36:57 +01:00
|
|
|
id := m.cm.Get(nodeID)
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RLock()
|
|
|
|
if r, ok := m.repoFiles[repo]; ok {
|
|
|
|
r.Update(id, files)
|
|
|
|
} else {
|
|
|
|
warnf("Index update from %s for nonexistant repo %q; dropping", nodeID, repo)
|
2013-12-28 14:10:36 +01:00
|
|
|
}
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RUnlock()
|
2014-01-09 10:59:09 +01:00
|
|
|
}
|
|
|
|
|
2014-01-20 22:22:27 +01:00
|
|
|
// Close removes the peer from the model and closes the underlying connection if possible.
|
2014-01-06 11:11:18 +01:00
|
|
|
// Implements the protocol.Model interface.
|
2013-12-31 03:21:57 +01:00
|
|
|
func (m *Model) Close(node string, err error) {
|
2014-03-09 09:15:36 +01:00
|
|
|
if debugNet {
|
|
|
|
dlog.Printf("%s: %v", node, err)
|
2014-02-09 23:13:06 +01:00
|
|
|
}
|
|
|
|
if err == protocol.ErrClusterHash {
|
2014-02-12 23:18:41 +01:00
|
|
|
warnf("Connection to %s closed due to mismatched cluster hash. Ensure that the configured cluster members are identical on both nodes.", node)
|
2014-02-13 12:41:25 +01:00
|
|
|
} else if err != io.EOF {
|
|
|
|
warnf("Connection to %s closed: %v", node, err)
|
2014-02-09 23:13:06 +01:00
|
|
|
}
|
|
|
|
|
2014-03-28 14:36:57 +01:00
|
|
|
cid := m.cm.Get(node)
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RLock()
|
|
|
|
for _, repo := range m.nodeRepos[node] {
|
|
|
|
m.repoFiles[repo].Replace(cid, nil)
|
|
|
|
}
|
|
|
|
m.rmut.RUnlock()
|
2014-03-28 14:36:57 +01:00
|
|
|
m.cm.Clear(node)
|
2014-01-20 22:22:27 +01:00
|
|
|
|
2014-01-18 04:06:44 +01:00
|
|
|
m.pmut.Lock()
|
2014-01-01 03:22:49 +01:00
|
|
|
conn, ok := m.rawConn[node]
|
2014-01-01 14:09:17 +01:00
|
|
|
if ok {
|
|
|
|
conn.Close()
|
2013-12-31 03:21:57 +01:00
|
|
|
}
|
2014-01-09 13:58:35 +01:00
|
|
|
delete(m.protoConn, node)
|
2014-01-01 03:22:49 +01:00
|
|
|
delete(m.rawConn, node)
|
2014-01-18 04:06:44 +01:00
|
|
|
m.pmut.Unlock()
|
2013-12-15 11:43:31 +01:00
|
|
|
}
|
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
// Request returns the specified data segment by reading it from local disk.
|
|
|
|
// Implements the protocol.Model interface.
|
2014-02-20 17:40:15 +01:00
|
|
|
func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]byte, error) {
|
2014-03-28 14:36:57 +01:00
|
|
|
// Verify that the requested file exists in the local model.
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RLock()
|
|
|
|
r, ok := m.repoFiles[repo]
|
|
|
|
m.rmut.RUnlock()
|
|
|
|
|
|
|
|
if !ok {
|
|
|
|
warnf("Request from %s for file %s in nonexistent repo %q", nodeID, name, repo)
|
|
|
|
return nil, ErrNoSuchFile
|
|
|
|
}
|
|
|
|
|
|
|
|
lf := r.Get(cid.LocalID, name)
|
2014-03-28 14:36:57 +01:00
|
|
|
if offset > lf.Size {
|
2014-02-20 17:40:15 +01:00
|
|
|
warnf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
|
2014-01-06 21:31:36 +01:00
|
|
|
return nil, ErrNoSuchFile
|
|
|
|
}
|
2014-03-29 18:53:48 +01:00
|
|
|
|
2014-03-16 08:14:55 +01:00
|
|
|
if lf.Suppressed {
|
2014-01-07 22:44:21 +01:00
|
|
|
return nil, ErrInvalid
|
|
|
|
}
|
2014-01-06 21:31:36 +01:00
|
|
|
|
2014-03-09 09:15:36 +01:00
|
|
|
if debugNet && nodeID != "<local>" {
|
2014-03-29 18:53:48 +01:00
|
|
|
dlog.Printf("REQ(in): %s: %q / %q o=%d s=%d", nodeID, repo, name, offset, size)
|
2013-12-15 11:43:31 +01:00
|
|
|
}
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RLock()
|
|
|
|
fn := filepath.Join(m.repoDirs[repo], name)
|
|
|
|
m.rmut.RUnlock()
|
2013-12-15 11:43:31 +01:00
|
|
|
fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer fd.Close()
|
|
|
|
|
|
|
|
buf := buffers.Get(int(size))
|
2014-01-09 16:35:49 +01:00
|
|
|
_, err = fd.ReadAt(buf, offset)
|
2013-12-15 11:43:31 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return buf, nil
|
|
|
|
}
|
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
// ReplaceLocal replaces the local repository index with the given list of files.
|
2014-03-29 18:53:48 +01:00
|
|
|
func (m *Model) ReplaceLocal(repo string, fs []scanner.File) {
|
|
|
|
m.rmut.RLock()
|
|
|
|
m.repoFiles[repo].ReplaceWithDelete(cid.LocalID, fs)
|
|
|
|
m.rmut.RUnlock()
|
2013-12-15 11:43:31 +01:00
|
|
|
}
|
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
func (m *Model) SeedLocal(repo string, fs []protocol.FileInfo) {
|
2014-03-28 14:36:57 +01:00
|
|
|
var sfs = make([]scanner.File, len(fs))
|
|
|
|
for i := 0; i < len(fs); i++ {
|
|
|
|
lamport.Default.Tick(fs[i].Version)
|
|
|
|
sfs[i] = fileFromFileInfo(fs[i])
|
2014-01-06 11:11:18 +01:00
|
|
|
}
|
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RLock()
|
|
|
|
m.repoFiles[repo].Replace(cid.LocalID, sfs)
|
|
|
|
m.rmut.RUnlock()
|
|
|
|
}
|
|
|
|
|
2014-04-01 23:18:32 +02:00
|
|
|
func (m *Model) CurrentRepoFile(repo string, file string) scanner.File {
|
|
|
|
m.rmut.RLock()
|
|
|
|
f := m.repoFiles[repo].Get(cid.LocalID, file)
|
|
|
|
m.rmut.RUnlock()
|
|
|
|
return f
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *Model) CurrentGlobalFile(repo string, file string) scanner.File {
|
|
|
|
m.rmut.RLock()
|
|
|
|
f := m.repoFiles[repo].GetGlobal(file)
|
|
|
|
m.rmut.RUnlock()
|
|
|
|
return f
|
|
|
|
}
|
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
type cFiler struct {
|
|
|
|
m *Model
|
|
|
|
r string
|
2014-01-06 11:11:18 +01:00
|
|
|
}
|
|
|
|
|
2014-03-16 08:14:55 +01:00
|
|
|
// Implements scanner.CurrentFiler
|
2014-03-29 18:53:48 +01:00
|
|
|
func (cf cFiler) CurrentFile(file string) scanner.File {
|
2014-04-01 23:18:32 +02:00
|
|
|
return cf.m.CurrentRepoFile(cf.r, file)
|
2014-03-16 08:14:55 +01:00
|
|
|
}
|
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
// ConnectedTo returns true if we are connected to the named node.
|
|
|
|
func (m *Model) ConnectedTo(nodeID string) bool {
|
2014-01-18 04:06:44 +01:00
|
|
|
m.pmut.RLock()
|
2014-01-09 13:58:35 +01:00
|
|
|
_, ok := m.protoConn[nodeID]
|
2014-01-18 04:06:44 +01:00
|
|
|
m.pmut.RUnlock()
|
2014-01-06 11:11:18 +01:00
|
|
|
return ok
|
|
|
|
}
|
|
|
|
|
|
|
|
// AddConnection adds a new peer connection to the model. An initial index will
|
|
|
|
// be sent to the connected peer, thereafter index updates whenever the local
|
|
|
|
// repository changes.
|
2014-03-28 14:36:57 +01:00
|
|
|
func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) {
|
2014-01-09 13:58:35 +01:00
|
|
|
nodeID := protoConn.ID()
|
2014-01-18 04:06:44 +01:00
|
|
|
m.pmut.Lock()
|
2014-03-23 08:45:05 +01:00
|
|
|
if _, ok := m.protoConn[nodeID]; ok {
|
|
|
|
panic("add existing node")
|
|
|
|
}
|
2014-01-09 13:58:35 +01:00
|
|
|
m.protoConn[nodeID] = protoConn
|
2014-03-23 08:45:05 +01:00
|
|
|
if _, ok := m.rawConn[nodeID]; ok {
|
|
|
|
panic("add existing node")
|
|
|
|
}
|
2014-01-09 13:58:35 +01:00
|
|
|
m.rawConn[nodeID] = rawConn
|
2014-01-18 04:06:44 +01:00
|
|
|
m.pmut.Unlock()
|
2014-01-06 11:11:18 +01:00
|
|
|
|
|
|
|
go func() {
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RLock()
|
|
|
|
repos := m.nodeRepos[nodeID]
|
|
|
|
m.rmut.RUnlock()
|
|
|
|
for _, repo := range repos {
|
|
|
|
idx := m.ProtocolIndex(repo)
|
|
|
|
if debugNet {
|
|
|
|
dlog.Printf("IDX(out/initial): %s: %q: %d files", nodeID, repo, len(idx))
|
|
|
|
}
|
|
|
|
protoConn.Index(repo, idx)
|
2014-03-09 09:15:36 +01:00
|
|
|
}
|
2014-01-06 11:11:18 +01:00
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
2014-01-18 04:06:44 +01:00
|
|
|
// ProtocolIndex returns the current local index in protocol data types.
|
2014-01-06 11:11:18 +01:00
|
|
|
// Must be called with the read lock held.
|
2014-03-29 18:53:48 +01:00
|
|
|
func (m *Model) ProtocolIndex(repo string) []protocol.FileInfo {
|
2014-01-06 11:11:18 +01:00
|
|
|
var index []protocol.FileInfo
|
2014-01-18 04:06:44 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RLock()
|
|
|
|
fs := m.repoFiles[repo].Have(cid.LocalID)
|
|
|
|
m.rmut.RUnlock()
|
2014-01-18 04:06:44 +01:00
|
|
|
|
2014-03-28 14:36:57 +01:00
|
|
|
for _, f := range fs {
|
2014-01-06 11:11:18 +01:00
|
|
|
mf := fileInfoFromFile(f)
|
2014-03-09 09:15:36 +01:00
|
|
|
if debugIdx {
|
2014-01-06 11:11:18 +01:00
|
|
|
var flagComment string
|
2014-01-07 22:44:21 +01:00
|
|
|
if mf.Flags&protocol.FlagDeleted != 0 {
|
2014-01-06 11:11:18 +01:00
|
|
|
flagComment = " (deleted)"
|
|
|
|
}
|
2014-03-29 18:53:48 +01:00
|
|
|
dlog.Printf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
|
2014-01-06 11:11:18 +01:00
|
|
|
}
|
|
|
|
index = append(index, mf)
|
|
|
|
}
|
2014-01-18 04:06:44 +01:00
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
return index
|
|
|
|
}
|
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
func (m *Model) updateLocal(repo string, f scanner.File) {
|
|
|
|
m.rmut.RLock()
|
|
|
|
m.repoFiles[repo].Update(cid.LocalID, []scanner.File{f})
|
|
|
|
m.rmut.RUnlock()
|
2014-03-28 14:36:57 +01:00
|
|
|
}
|
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
func (m *Model) requestGlobal(nodeID, repo, name string, offset int64, size int, hash []byte) ([]byte, error) {
|
2014-01-18 04:06:44 +01:00
|
|
|
m.pmut.RLock()
|
2014-01-09 13:58:35 +01:00
|
|
|
nc, ok := m.protoConn[nodeID]
|
2014-01-18 04:06:44 +01:00
|
|
|
m.pmut.RUnlock()
|
|
|
|
|
2014-01-06 11:11:18 +01:00
|
|
|
if !ok {
|
|
|
|
return nil, fmt.Errorf("requestGlobal: no such node: %s", nodeID)
|
|
|
|
}
|
|
|
|
|
2014-03-09 09:15:36 +01:00
|
|
|
if debugNet {
|
2014-03-29 18:53:48 +01:00
|
|
|
dlog.Printf("REQ(out): %s: %q / %q o=%d s=%d h=%x", nodeID, repo, name, offset, size, hash)
|
2014-01-06 11:11:18 +01:00
|
|
|
}
|
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
return nc.Request(repo, name, offset, size)
|
2014-01-06 11:11:18 +01:00
|
|
|
}
|
|
|
|
|
2013-12-24 21:21:03 +01:00
|
|
|
func (m *Model) broadcastIndexLoop() {
|
2014-03-29 18:53:48 +01:00
|
|
|
var lastChange = map[string]uint64{}
|
2013-12-24 21:21:03 +01:00
|
|
|
for {
|
2014-03-28 14:36:57 +01:00
|
|
|
time.Sleep(5 * time.Second)
|
2014-01-18 04:06:44 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
m.pmut.RLock()
|
|
|
|
m.rmut.RLock()
|
|
|
|
|
|
|
|
for repo, fs := range m.repoFiles {
|
|
|
|
c := fs.Changes(cid.LocalID)
|
|
|
|
if c == lastChange[repo] {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
lastChange[repo] = c
|
|
|
|
|
|
|
|
idx := m.ProtocolIndex(repo)
|
|
|
|
m.saveIndex(repo, confDir, idx)
|
|
|
|
|
|
|
|
var indexWg sync.WaitGroup
|
|
|
|
for _, nodeID := range m.repoNodes[repo] {
|
|
|
|
if conn, ok := m.protoConn[nodeID]; ok {
|
|
|
|
indexWg.Add(1)
|
|
|
|
if debugNet {
|
|
|
|
dlog.Printf("IDX(out/loop): %s: %d files", nodeID, len(idx))
|
|
|
|
}
|
|
|
|
go func() {
|
|
|
|
conn.Index(repo, idx)
|
|
|
|
indexWg.Done()
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
indexWg.Wait()
|
2013-12-15 11:43:31 +01:00
|
|
|
}
|
2014-01-09 16:35:49 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
m.rmut.RUnlock()
|
|
|
|
m.pmut.RUnlock()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *Model) AddRepo(id, dir string, nodes []NodeConfiguration) {
|
|
|
|
if m.started {
|
|
|
|
panic("cannot add repo to started model")
|
|
|
|
}
|
|
|
|
if len(id) == 0 {
|
|
|
|
panic("cannot add empty repo id")
|
|
|
|
}
|
|
|
|
|
|
|
|
m.rmut.Lock()
|
|
|
|
m.repoDirs[id] = dir
|
|
|
|
m.repoFiles[id] = files.NewSet()
|
2013-12-15 11:43:31 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
m.repoNodes[id] = make([]string, len(nodes))
|
|
|
|
for i, node := range nodes {
|
|
|
|
m.repoNodes[id][i] = node.NodeID
|
|
|
|
m.nodeRepos[node.NodeID] = append(m.nodeRepos[node.NodeID], id)
|
|
|
|
}
|
2014-01-23 22:20:15 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
m.addedRepo = true
|
|
|
|
m.rmut.Unlock()
|
|
|
|
}
|
2014-01-23 22:20:15 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
func (m *Model) ScanRepos() {
|
|
|
|
m.rmut.RLock()
|
|
|
|
for repo := range m.repoDirs {
|
|
|
|
m.ScanRepo(repo)
|
|
|
|
}
|
|
|
|
m.rmut.RUnlock()
|
|
|
|
}
|
2013-12-15 11:43:31 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
func (m *Model) ScanRepo(repo string) {
|
|
|
|
sup := &suppressor{threshold: int64(cfg.Options.MaxChangeKbps)}
|
|
|
|
w := &scanner.Walker{
|
2014-04-08 13:45:18 +02:00
|
|
|
Dir: m.repoDirs[repo],
|
|
|
|
IgnoreFile: ".stignore",
|
|
|
|
BlockSize: BlockSize,
|
|
|
|
TempNamer: defTempNamer,
|
|
|
|
Suppressor: sup,
|
|
|
|
CurrentFiler: cFiler{m, repo},
|
2014-03-29 18:53:48 +01:00
|
|
|
}
|
|
|
|
fs, _ := w.Walk()
|
|
|
|
m.ReplaceLocal(repo, fs)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *Model) SaveIndexes(dir string) {
|
|
|
|
m.rmut.RLock()
|
|
|
|
for repo := range m.repoDirs {
|
|
|
|
fs := m.ProtocolIndex(repo)
|
|
|
|
m.saveIndex(repo, dir, fs)
|
|
|
|
}
|
|
|
|
m.rmut.RUnlock()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *Model) LoadIndexes(dir string) {
|
|
|
|
m.rmut.RLock()
|
|
|
|
for repo := range m.repoDirs {
|
|
|
|
fs := m.loadIndex(repo, dir)
|
|
|
|
m.SeedLocal(repo, fs)
|
|
|
|
}
|
|
|
|
m.rmut.RUnlock()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *Model) saveIndex(repo string, dir string, fs []protocol.FileInfo) {
|
|
|
|
id := fmt.Sprintf("%x", sha1.Sum([]byte(m.repoDirs[repo])))
|
|
|
|
name := id + ".idx.gz"
|
|
|
|
name = filepath.Join(dir, name)
|
|
|
|
|
|
|
|
idxf, err := os.Create(name + ".tmp")
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
2014-01-18 04:06:44 +01:00
|
|
|
|
2014-03-29 18:53:48 +01:00
|
|
|
gzw := gzip.NewWriter(idxf)
|
|
|
|
|
|
|
|
protocol.IndexMessage{
|
|
|
|
Repository: repo,
|
|
|
|
Files: fs,
|
|
|
|
}.EncodeXDR(gzw)
|
|
|
|
gzw.Close()
|
|
|
|
idxf.Close()
|
|
|
|
|
|
|
|
Rename(name+".tmp", name)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *Model) loadIndex(repo string, dir string) []protocol.FileInfo {
|
|
|
|
id := fmt.Sprintf("%x", sha1.Sum([]byte(m.repoDirs[repo])))
|
|
|
|
name := id + ".idx.gz"
|
|
|
|
name = filepath.Join(dir, name)
|
|
|
|
|
|
|
|
idxf, err := os.Open(name)
|
|
|
|
if err != nil {
|
|
|
|
return nil
|
2014-01-09 16:35:49 +01:00
|
|
|
}
|
2014-03-29 18:53:48 +01:00
|
|
|
defer idxf.Close()
|
|
|
|
|
|
|
|
gzr, err := gzip.NewReader(idxf)
|
|
|
|
if err != nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
defer gzr.Close()
|
|
|
|
|
|
|
|
var im protocol.IndexMessage
|
|
|
|
err = im.DecodeXDR(gzr)
|
|
|
|
if err != nil || im.Repository != repo {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return im.Files
|
2014-01-09 16:35:49 +01:00
|
|
|
}
|
|
|
|
|
2014-03-08 23:02:01 +01:00
|
|
|
func fileFromFileInfo(f protocol.FileInfo) scanner.File {
|
|
|
|
var blocks = make([]scanner.Block, len(f.Blocks))
|
2014-01-09 16:35:49 +01:00
|
|
|
var offset int64
|
2014-01-09 13:58:35 +01:00
|
|
|
for i, b := range f.Blocks {
|
2014-03-08 23:02:01 +01:00
|
|
|
blocks[i] = scanner.Block{
|
2013-12-28 14:10:36 +01:00
|
|
|
Offset: offset,
|
2014-01-09 16:35:49 +01:00
|
|
|
Size: b.Size,
|
2013-12-28 14:10:36 +01:00
|
|
|
Hash: b.Hash,
|
2014-01-09 13:58:35 +01:00
|
|
|
}
|
2014-01-09 16:35:49 +01:00
|
|
|
offset += int64(b.Size)
|
2013-12-28 14:10:36 +01:00
|
|
|
}
|
2014-03-08 23:02:01 +01:00
|
|
|
return scanner.File{
|
2014-03-28 14:36:57 +01:00
|
|
|
// Name is with native separator and normalization
|
|
|
|
Name: filepath.FromSlash(f.Name),
|
2014-03-16 08:14:55 +01:00
|
|
|
Size: offset,
|
|
|
|
Flags: f.Flags &^ protocol.FlagInvalid,
|
|
|
|
Modified: f.Modified,
|
|
|
|
Version: f.Version,
|
|
|
|
Blocks: blocks,
|
|
|
|
Suppressed: f.Flags&protocol.FlagInvalid != 0,
|
2013-12-30 01:49:40 +01:00
|
|
|
}
|
2013-12-28 14:10:36 +01:00
|
|
|
}
|
2013-12-30 02:33:57 +01:00
|
|
|
|
2014-03-08 23:02:01 +01:00
|
|
|
func fileInfoFromFile(f scanner.File) protocol.FileInfo {
|
2014-01-09 13:58:35 +01:00
|
|
|
var blocks = make([]protocol.BlockInfo, len(f.Blocks))
|
|
|
|
for i, b := range f.Blocks {
|
|
|
|
blocks[i] = protocol.BlockInfo{
|
2014-01-09 16:35:49 +01:00
|
|
|
Size: b.Size,
|
|
|
|
Hash: b.Hash,
|
2014-01-09 13:58:35 +01:00
|
|
|
}
|
2013-12-30 02:33:57 +01:00
|
|
|
}
|
2014-03-16 08:14:55 +01:00
|
|
|
pf := protocol.FileInfo{
|
2014-03-28 14:36:57 +01:00
|
|
|
Name: filepath.ToSlash(f.Name),
|
2013-12-30 02:33:57 +01:00
|
|
|
Flags: f.Flags,
|
2014-01-09 16:35:49 +01:00
|
|
|
Modified: f.Modified,
|
2014-01-07 22:44:21 +01:00
|
|
|
Version: f.Version,
|
2013-12-30 02:33:57 +01:00
|
|
|
Blocks: blocks,
|
|
|
|
}
|
2014-03-16 08:14:55 +01:00
|
|
|
if f.Suppressed {
|
|
|
|
pf.Flags |= protocol.FlagInvalid
|
|
|
|
}
|
|
|
|
return pf
|
2013-12-30 02:33:57 +01:00
|
|
|
}
|