lib/ur: Implement crash (panic) reporting (fixes #959) (#5702)

* lib/ur: Implement crash (panic) reporting (fixes #959) This implements a simple crash reporting method. It piggybacks on the panic log files created by the monitor process, picking these up and uploading them from the usage reporting routine. A new config value points to the crash receiver base URL, which defaults to "https://crash.syncthing.net/newcrash" (following the pattern of "https://data.syncthing.net/newdata" for usage reports, but allowing us to separate the service as required).
2019-06-11 08:19:11 +02:00 · 2019-06-11 08:19:11 +02:00 · 42ce6be9b9
parent 93e57bd357
commit 42ce6be9b9
19 changed files with 1925 additions and 19 deletions
--- a/cmd/stcrashreceiver/_testdata/panic.log
+++ b/cmd/stcrashreceiver/_testdata/panic.log
--- a/cmd/stcrashreceiver/sentry.go
+++ b/cmd/stcrashreceiver/sentry.go
@ -0,0 +1,160 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"regexp"
+	"strings"
+
+	raven "github.com/getsentry/raven-go"
+	"github.com/maruel/panicparse/stack"
+)
+
+const reportServer = "https://crash.syncthing.net/report/"
+
+func sendReport(dsn, path string, report []byte) error {
+	pkt, err := parseReport(path, report)
+	if err != nil {
+		return err
+	}
+
+	cli, err := raven.New(dsn)
+	if err != nil {
+		return err
+	}
+
+	// The client sets release and such on the packet before sending, in the
+	// misguided idea that it knows this better than than the packet we give
+	// it. So we copy the values from the packet to the client first...
+	cli.SetRelease(pkt.Release)
+	cli.SetEnvironment(pkt.Environment)
+
+	_, errC := cli.Capture(pkt, nil)
+	return <-errC
+}
+
+func parseReport(path string, report []byte) (*raven.Packet, error) {
+	parts := bytes.SplitN(report, []byte("\n"), 2)
+	if len(parts) != 2 {
+		return nil, errors.New("no first line")
+	}
+
+	version, err := parseVersion(string(parts[0]))
+	if err != nil {
+		return nil, err
+	}
+	report = parts[1]
+
+	foundPanic := false
+	var subjectLine []byte
+	for {
+		parts = bytes.SplitN(report, []byte("\n"), 2)
+		if len(parts) != 2 {
+			return nil, errors.New("no panic line found")
+		}
+
+		line := parts[0]
+		report = parts[1]
+
+		if foundPanic {
+			// The previous line was our "Panic at ..." header. We are now
+			// at the beginning of the real panic trace and this is our
+			// subject line.
+			subjectLine = line
+			break
+		} else if bytes.HasPrefix(line, []byte("Panic at")) {
+			foundPanic = true
+		}
+	}
+
+	r := bytes.NewReader(report)
+	ctx, err := stack.ParseDump(r, ioutil.Discard, false)
+	if err != nil {
+		return nil, err
+	}
+
+	var trace raven.Stacktrace
+	for _, gr := range ctx.Goroutines {
+		if gr.First {
+			trace.Frames = make([]*raven.StacktraceFrame, len(gr.Stack.Calls))
+			for i, sc := range gr.Stack.Calls {
+				trace.Frames[len(trace.Frames)-1-i] = &raven.StacktraceFrame{
+					Function: sc.Func.Name(),
+					Module:   sc.Func.PkgName(),
+					Filename: sc.SrcPath,
+					Lineno:   sc.Line,
+				}
+			}
+			break
+		}
+	}
+
+	pkt := &raven.Packet{
+		Message:  string(subjectLine),
+		Platform: "go",
+		Release:  version.tag,
+		Tags: raven.Tags{
+			raven.Tag{Key: "version", Value: version.version},
+			raven.Tag{Key: "tag", Value: version.tag},
+			raven.Tag{Key: "commit", Value: version.commit},
+			raven.Tag{Key: "codename", Value: version.codename},
+			raven.Tag{Key: "runtime", Value: version.runtime},
+			raven.Tag{Key: "goos", Value: version.goos},
+			raven.Tag{Key: "goarch", Value: version.goarch},
+			raven.Tag{Key: "builder", Value: version.builder},
+		},
+		Extra: raven.Extra{
+			"url": reportServer + path,
+		},
+		Interfaces: []raven.Interface{&trace},
+	}
+
+	return pkt, nil
+}
+
+// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC
+var longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)`)
+
+type version struct {
+	version  string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep"
+	tag      string // "v1.1.4-rc.1"
+	commit   string // "6aaae618", blank when absent
+	codename string // "Erbium Earthworm"
+	runtime  string // "go1.12.5"
+	goos     string // "darwin"
+	goarch   string // "amd64"
+	builder  string // "jb@kvin.kastelo.net"
+}
+
+func parseVersion(line string) (version, error) {
+	m := longVersionRE.FindStringSubmatch(line)
+	if len(m) == 0 {
+		return version{}, errors.New("unintelligeble version string")
+	}
+
+	v := version{
+		version:  m[1],
+		codename: m[2],
+		runtime:  m[3],
+		goos:     m[4],
+		goarch:   m[5],
+		builder:  m[6],
+	}
+	parts := strings.Split(v.version, "+")
+	v.tag = parts[0]
+	if len(parts) > 1 {
+		fields := strings.Split(parts[1], "-")
+		if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") {
+			v.commit = fields[1][1:]
+		}
+	}
+
+	return v, nil
+}
--- a/cmd/stcrashreceiver/sentry_test.go
+++ b/cmd/stcrashreceiver/sentry_test.go
@ -0,0 +1,64 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+)
+
+func TestParseVersion(t *testing.T) {
+	cases := []struct {
+		longVersion string
+		parsed      version
+	}{
+		{
+			longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`,
+			parsed: version{
+				version:  "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep",
+				tag:      "v1.1.4-rc.1",
+				commit:   "6aaae618",
+				codename: "Erbium Earthworm",
+				runtime:  "go1.12.5",
+				goos:     "darwin",
+				goarch:   "amd64",
+				builder:  "jb@kvin.kastelo.net",
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		v, err := parseVersion(tc.longVersion)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		if v != tc.parsed {
+			t.Error(v)
+		}
+	}
+}
+
+func TestParseReport(t *testing.T) {
+	bs, err := ioutil.ReadFile("_testdata/panic.log")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pkt, err := parseReport("1/2/345", bs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bs, err = pkt.JSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Printf("%s\n", bs)
+}
--- a/cmd/stcrashreceiver/stcrashreceiver.go
+++ b/cmd/stcrashreceiver/stcrashreceiver.go
@ -0,0 +1,135 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Command stcrashreceiver is a trivial HTTP server that allows two things:
+//
+// - uploading files (crash reports) named like a SHA256 hash using a PUT request
+// - checking whether such file exists using a HEAD request
+//
+// Typically this should be deployed behind something that manages HTTPS.
+package main
+
+import (
+	"flag"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+const maxRequestSize = 1 << 20 // 1 MiB
+
+func main() {
+	dir := flag.String("dir", ".", "Directory to store reports in")
+	dsn := flag.String("dsn", "", "Sentry DSN")
+	listen := flag.String("listen", ":22039", "HTTP listen address")
+	flag.Parse()
+
+	cr := &crashReceiver{
+		dir: *dir,
+		dsn: *dsn,
+	}
+
+	log.SetOutput(os.Stdout)
+	if err := http.ListenAndServe(*listen, cr); err != nil {
+		log.Fatalln("HTTP serve:", err)
+	}
+}
+
+type crashReceiver struct {
+	dir string
+	dsn string
+}
+
+func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// The final path component should be a SHA256 hash in hex, so 64 hex
+	// characters. We don't care about case on the request but use lower
+	// case internally.
+	base := strings.ToLower(path.Base(req.URL.Path))
+	if len(base) != 64 {
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+	for _, c := range base {
+		if c >= 'a' && c <= 'f' {
+			continue
+		}
+		if c >= '0' && c <= '9' {
+			continue
+		}
+		http.Error(w, "Bad request", http.StatusBadRequest)
+		return
+	}
+
+	switch req.Method {
+	case http.MethodHead:
+		r.serveHead(base, w, req)
+	case http.MethodPut:
+		r.servePut(base, w, req)
+	default:
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// serveHead responds to HEAD requests by checking if the named report
+// already exists in the system.
+func (r *crashReceiver) serveHead(base string, w http.ResponseWriter, _ *http.Request) {
+	path := filepath.Join(r.dirFor(base), base)
+	if _, err := os.Lstat(path); err != nil {
+		http.Error(w, "Not found", http.StatusNotFound)
+	}
+	// 200 OK
+}
+
+// servePut accepts and stores the given report.
+func (r *crashReceiver) servePut(base string, w http.ResponseWriter, req *http.Request) {
+	path := filepath.Join(r.dirFor(base), base)
+	fullPath := filepath.Join(r.dir, path)
+
+	// Ensure the destination directory exists
+	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+		log.Printf("Creating directory for report %s: %v", base, err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Read at most maxRequestSize of report data.
+	log.Println("Receiving report", base)
+	lr := io.LimitReader(req.Body, maxRequestSize)
+	bs, err := ioutil.ReadAll(lr)
+	if err != nil {
+		log.Println("Reading report:", err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Create an output file
+	err = ioutil.WriteFile(fullPath, bs, 0644)
+	if err != nil {
+		log.Printf("Creating file for report %s: %v", base, err)
+		http.Error(w, "Internal server error", http.StatusInternalServerError)
+		return
+	}
+
+	// Send the report to Sentry
+	if r.dsn != "" {
+		go func() {
+			// There's no need for the client to have to wait for this part.
+			if err := sendReport(r.dsn, path, bs); err != nil {
+				log.Println("Failed to send report:", err)
+			}
+		}()
+	}
+}
+
+// 01234567890abcdef... => 01/23
+func (r *crashReceiver) dirFor(base string) string {
+	return filepath.Join(base[0:2], base[2:4])
+}
--- a/cmd/syncthing/crash_reporting.go
+++ b/cmd/syncthing/crash_reporting.go
@ -0,0 +1,152 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/sha256"
+)
+
+const (
+	headRequestTimeout = 10 * time.Second
+	putRequestTimeout  = time.Minute
+)
+
+// uploadPanicLogs attempts to upload all the panic logs in the named
+// directory to the crash reporting server as urlBase. Uploads are attempted
+// with the newest log first.
+//
+// This can can block for a long time. The context can set a final deadline
+// for this.
+func uploadPanicLogs(ctx context.Context, urlBase, dir string) {
+	files, err := filepath.Glob(filepath.Join(dir, "panic-*.log"))
+	if err != nil {
+		l.Warnln("Failed to list panic logs:", err)
+		return
+	}
+
+	sort.Sort(sort.Reverse(sort.StringSlice(files)))
+	for _, file := range files {
+		if strings.Contains(file, ".reported.") {
+			// We've already sent this file. It'll be cleaned out at some
+			// point.
+			continue
+		}
+
+		if err := uploadPanicLog(ctx, urlBase, file); err != nil {
+			l.Warnln("Reporting crash:", err)
+		} else {
+			// Rename the log so we don't have to try to report it again. This
+			// succeeds, or it does not. There is no point complaining about it.
+			_ = os.Rename(file, strings.Replace(file, ".log", ".reported.log", 1))
+		}
+	}
+}
+
+// uploadPanicLog attempts to upload the named panic log to the crash
+// reporting server at urlBase. The panic ID is constructed as the sha256 of
+// the log contents. A HEAD request is made to see if the log has already
+// been reported. If not, a PUT is made with the log contents.
+func uploadPanicLog(ctx context.Context, urlBase, file string) error {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return err
+	}
+
+	// Remove log lines, for privacy.
+	data = filterLogLines(data)
+
+	hash := fmt.Sprintf("%x", sha256.Sum256(data))
+	l.Infof("Reporting crash found in %s (report ID %s) ...\n", filepath.Base(file), hash[:8])
+
+	url := fmt.Sprintf("%s/%s", urlBase, hash)
+	headReq, err := http.NewRequest(http.MethodHead, url, nil)
+	if err != nil {
+		return err
+	}
+
+	// Set a reasonable timeout on the HEAD request
+	headCtx, headCancel := context.WithTimeout(ctx, headRequestTimeout)
+	defer headCancel()
+	headReq = headReq.WithContext(headCtx)
+
+	resp, err := http.DefaultClient.Do(headReq)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode == http.StatusOK {
+		// It's known, we're done
+		return nil
+	}
+
+	putReq, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	// Set a reasonable timeout on the PUT request
+	putCtx, putCancel := context.WithTimeout(ctx, putRequestTimeout)
+	defer putCancel()
+	putReq = putReq.WithContext(putCtx)
+
+	resp, err = http.DefaultClient.Do(putReq)
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("upload: %s", resp.Status)
+	}
+
+	return nil
+}
+
+// filterLogLines returns the data without any log lines between the first
+// line and the panic trace. This is done in-place: the original data slice
+// is destroyed.
+func filterLogLines(data []byte) []byte {
+	filtered := data[:0]
+	matched := false
+	for _, line := range bytes.Split(data, []byte("\n")) {
+		switch {
+		case !matched && bytes.HasPrefix(line, []byte("Panic ")):
+			// This begins the panic trace, set the matched flag and append.
+			matched = true
+			fallthrough
+		case len(filtered) == 0 || matched:
+			// This is the first line or inside the panic trace.
+			if len(filtered) > 0 {
+				// We add the newline before rather than after because
+				// bytes.Split sees the \n as *separator* and not line
+				// ender, so ir will generate a last empty line that we
+				// don't really want. (We want to keep blank lines in the
+				// middle of the trace though.)
+				filtered = append(filtered, '\n')
+			}
+			// Remove the device ID prefix. The "plus two" stuff is because
+			// the line will look like "[foo] whatever" and the end variable
+			// will end up pointing at the ] and we want to step over that
+			// and the following space.
+			if end := bytes.Index(line, []byte("]")); end > 1 && end < len(line)-2 && bytes.HasPrefix(line, []byte("[")) {
+				line = line[end+2:]
+			}
+			filtered = append(filtered, line...)
+		}
+	}
+	return filtered
+}
--- a/cmd/syncthing/crash_reporting_test.go
+++ b/cmd/syncthing/crash_reporting_test.go
@ -0,0 +1,37 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestFilterLogLines(t *testing.T) {
+	in := []byte(`[ABCD123] syncthing version whatever
+here is more log data
+and more
+...
+and some more
+yet more
+Panic detected at like right now
+here is panic data
+and yet more panic stuff
+`)
+
+	filtered := []byte(`syncthing version whatever
+Panic detected at like right now
+here is panic data
+and yet more panic stuff
+`)
+
+	result := filterLogLines(in)
+	if !bytes.Equal(result, filtered) {
+		t.Logf("%q\n", result)
+		t.Error("it should have been filtered")
+	}
+}
--- a/cmd/syncthing/monitor.go
+++ b/cmd/syncthing/monitor.go
@ -8,6 +8,7 @@ package main

 import (
 	"bufio"
+	"context"
 	"io"
 	"os"
 	"os/exec"
@ -33,6 +34,8 @@ const (
 	loopThreshold         = 60 * time.Second
 	logFileAutoCloseDelay = 5 * time.Second
 	logFileMaxOpenTime    = time.Minute
+	panicUploadMaxWait    = 30 * time.Second
+	panicUploadNoticeWait = 10 * time.Second
 )

 func monitorMain(runtimeOptions RuntimeOptions) {
@ -72,6 +75,8 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 	childEnv := childEnv()
 	first := true
 	for {
+		maybeReportPanics()
+
 		if t := time.Since(restarts[0]); t < loopThreshold {
 			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
 			os.Exit(exitError)
@ -173,6 +178,13 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 	br := bufio.NewReader(stderr)

 	var panicFd *os.File
+	defer func() {
+		if panicFd != nil {
+			_ = panicFd.Close()
+			maybeReportPanics()
+		}
+	}()
+
 	for {
 		line, err := br.ReadString('\n')
 		if err != nil {
@ -430,3 +442,39 @@ func childEnv() []string {
 	env = append(env, "STMONITORED=yes")
 	return env
 }
+
+// maybeReportPanics tries to figure out if crash reporting is on or off,
+// and reports any panics it can find if it's enabled. We spend at most
+// panicUploadMaxWait uploading panics...
+func maybeReportPanics() {
+	// Try to get a config to see if/where panics should be reported.
+	cfg, err := loadOrDefaultConfig()
+	if err != nil {
+		l.Warnln("Couldn't load config; not reporting crash")
+		return
+	}
+
+	// Bail if we're not supposed to report panics.
+	opts := cfg.Options()
+	if !opts.CREnabled {
+		return
+	}
+
+	// Set up a timeout on the whole operation.
+	ctx, cancel := context.WithTimeout(context.Background(), panicUploadMaxWait)
+	defer cancel()
+
+	// Print a notice if the upload takes a long time.
+	go func() {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(panicUploadNoticeWait):
+			l.Warnln("Uploading crash reports is taking a while, please wait...")
+		}
+	}()
+
+	// Report the panics.
+	dir := locations.GetBaseDir(locations.ConfigBaseDir)
+	uploadPanicLogs(ctx, opts.CRURL, dir)
+}
--- a/go.mod
+++ b/go.mod
@ -8,9 +8,11 @@ require (
 	github.com/calmh/du v1.0.1
 	github.com/calmh/xdr v1.1.0
 	github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d
+	github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 // indirect
 	github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5
 	github.com/d4l3k/messagediff v1.2.1
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
+	github.com/getsentry/raven-go v0.2.0
 	github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d
 	github.com/gogo/protobuf v1.2.1
 	github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4
@ -20,6 +22,7 @@ require (
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/lib/pq v1.1.1
 	github.com/lucas-clemente/quic-go v0.11.2
+	github.com/maruel/panicparse v1.2.1
 	github.com/mattn/go-isatty v0.0.7
 	github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338
 	github.com/oschwald/geoip2-golang v1.3.0
--- a/go.sum
+++ b/go.sum
@ -19,6 +19,8 @@ github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE=
 github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg=
 github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d h1:As4937T5NVbJ/DmZT9z33pyLEprMd6CUSfhbmMY57Io=
 github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d/go.mod h1:3FK1bMar37f7jqVY7q/63k3OMX1c47pGCufzt3X0sYE=
+github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 h1:UNOqI3EKhvbqV8f1Vm3NIwkrhq388sGCeAH2Op7w0rc=
+github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
@ -34,6 +36,8 @@ github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWT
 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
+github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
@ -78,10 +82,15 @@ github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFM
 github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
 github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
 github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
+github.com/maruel/panicparse v1.2.1 h1:mNlHGiakrixj+AwF/qRpTwnj+zsWYPRLQ7wRqnJsfO0=
+github.com/maruel/panicparse v1.2.1/go.mod h1:vszMjr5QQ4F5FSRfraldcIA/BCw5xrdLL+zEcU2nRBs=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338 h1:USW1+zAUkUSvk097CAX/i8KR3r6f+DHNhk6Xe025Oyw=
 github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
--- a/gui/default/syncthing/core/notifications.html
+++ b/gui/default/syncthing/core/notifications.html
@ -69,3 +69,50 @@
    </div>
  </div>
 </notification>
+
+<notification id="crAutoEnabled">
+  <div class="panel panel-success">
+    <div class="panel-heading">
+      <h3 class="panel-title"><span class="fas fa-bolt"></span>&ensp;<span translate>Automatic Crash Reporting</span></h3>
+    </div>
+    <div class="panel-body">
+      <p translate>Syncthing now supports automatically reporting crashes to the developers. This feature is enabled by default.</p>
+      <p><a href="https://docs.syncthing.net/users/crashrep.html"><span class="fas fa-info-circle"></span>&nbsp;<span translate>Learn more</span></a></p>
+    </div>
+    <div class="panel-footer clearfix">
+      <div class="pull-right">
+        <button type="button" class="btn btn-danger" ng-click="setCrashReportingEnabled(false); dismissNotification('crAutoEnabled')">
+          <span class="fas fa-times"></span>&nbsp;<span translate>Disable Crash Reporting</span>
+        </button>
+        <button type="button" class="btn btn-default" ng-click="dismissNotification('crAutoEnabled')">
+          <span class="fas fa-check"></span>&nbsp;<span translate>OK</span>
+        </button>
+      </div>
+      <div class="clearfix"></div>
+    </div>
+  </div>
+</notification>
+
+<notification id="crAutoDisabled">
+  <div class="panel panel-success">
+    <div class="panel-heading">
+      <h3 class="panel-title"><span class="fas fa-bolt"></span>&ensp;<span translate>Automatic Crash Reporting</span></h3>
+    </div>
+    <div class="panel-body">
+      <p translate>Syncthing now supports automatically reporting crashes to the developers. This feature is enabled by default.</p>
+      <p translate>However, your current settings indicate you might not want it enabled. We have disabled automatic crash reporting for you.</p>
+      <p><a href="https://docs.syncthing.net/users/crashrep.html"><span class="fas fa-info-circle"></span>&nbsp;<span translate>Learn more</span></a></p>
+    </div>
+    <div class="panel-footer clearfix">
+      <div class="pull-right">
+        <button type="button" class="btn btn-success" ng-click="setCrashReportingEnabled(true); dismissNotification('crAutoDisabled')">
+          <span class="fas fa-check"></span>&nbsp;<span translate>Enable Crash Reporting</span>
+        </button>
+        <button type="button" class="btn btn-default" ng-click="dismissNotification('crAutoDisabled')">
+          <span class="fas fa-times"></span>&nbsp;<span translate>OK</span>
+        </button>
+      </div>
+      <div class="clearfix"></div>
+    </div>
+  </div>
+</notification>
--- a/gui/default/syncthing/core/syncthingController.js
+++ b/gui/default/syncthing/core/syncthingController.js
@ -2426,4 +2426,9 @@ angular.module('syncthing.core')
            var err = status.error.replace(/.+: /, '');
            return err + " (" + time + ")";
        }
+
+        $scope.setCrashReportingEnabled = function (enabled) {
+            $scope.config.options.crashReportingEnabled = enabled;
+            $scope.saveConfig();
+        };
    });
--- a/lib/config/config.go
+++ b/lib/config/config.go
@ -29,7 +29,7 @@ import (

 const (
 	OldestHandledVersion = 10
-	CurrentVersion       = 28
+	CurrentVersion       = 29
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )

--- a/lib/config/config_test.go
+++ b/lib/config/config_test.go
@ -69,6 +69,8 @@ func TestDefaultValues(t *testing.T) {
 		UnackedNotificationIDs:  []string{},
 		DefaultFolderPath:       "~",
 		SetLowPriority:          true,
+		CRURL:                   "https://crash.syncthing.net/newcrash",
+		CREnabled:               true,
 		StunKeepaliveStartS:     180,
 		StunKeepaliveMinS:       20,
 		StunServers:             []string{"default"},
@ -203,7 +205,8 @@ func TestOverriddenValues(t *testing.T) {
 		ProgressUpdateIntervalS: 10,
 		LimitBandwidthInLan:     true,
 		MinHomeDiskFree:         Size{5.2, "%"},
-		URSeen:                  2,
+		URSeen:                  8,
+		URAccepted:              4,
 		URURL:                   "https://localhost/newdata",
 		URInitialDelayS:         800,
 		URPostInsecurely:        true,
@ -211,15 +214,14 @@ func TestOverriddenValues(t *testing.T) {
 		AlwaysLocalNets:         []string{},
 		OverwriteRemoteDevNames: true,
 		TempIndexMinBlocks:      100,
-		UnackedNotificationIDs: []string{
-			"channelNotification",   // added in 17->18 migration
-			"fsWatcherNotification", // added in 27->28 migration
-		},
-		DefaultFolderPath:   "/media/syncthing",
-		SetLowPriority:      false,
-		StunKeepaliveStartS: 9000,
-		StunKeepaliveMinS:   900,
-		StunServers:         []string{"foo"},
+		UnackedNotificationIDs:  []string{"asdfasdf"},
+		DefaultFolderPath:       "/media/syncthing",
+		SetLowPriority:          false,
+		CRURL:                   "https://localhost/newcrash",
+		CREnabled:               false,
+		StunKeepaliveStartS:     9000,
+		StunKeepaliveMinS:       900,
+		StunServers:             []string{"foo"},
 	}

 	os.Unsetenv("STNOUPGRADE")
--- a/lib/config/migrations.go
+++ b/lib/config/migrations.go
@ -25,6 +25,7 @@ import (
 // update the config version. The order of migrations doesn't matter here,
 // put the newest on top for readability.
 var migrations = migrationSet{
+	{29, migrateToConfigV29},
 	{28, migrateToConfigV28},
 	{27, migrateToConfigV27},
 	{26, nil}, // triggers database update
@ -83,6 +84,19 @@ func (m migration) apply(cfg *Configuration) {
 	cfg.Version = m.targetVersion
 }

+func migrateToConfigV29(cfg *Configuration) {
+	// The new crash reporting option should follow the state of global
+	// discovery / usage reporting, and we should display an appropriate
+	// notification.
+	if cfg.Options.GlobalAnnEnabled || cfg.Options.URAccepted > 0 {
+		cfg.Options.CREnabled = true
+		cfg.Options.UnackedNotificationIDs = append(cfg.Options.UnackedNotificationIDs, "crAutoEnabled")
+	} else {
+		cfg.Options.CREnabled = false
+		cfg.Options.UnackedNotificationIDs = append(cfg.Options.UnackedNotificationIDs, "crAutoDisabled")
+	}
+}
+
 func migrateToConfigV28(cfg *Configuration) {
 	// Show a notification about enabling filesystem watching
 	cfg.Options.UnackedNotificationIDs = append(cfg.Options.UnackedNotificationIDs, "fsWatcherNotification")
--- a/lib/config/migrations_test.go
+++ b/lib/config/migrations_test.go
@ -0,0 +1,34 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package config
+
+import "testing"
+
+func TestMigrateCrashReporting(t *testing.T) {
+	// When migrating from pre-crash-reporting configs, crash reporting is
+	// enabled if global discovery is enabled or if usage reporting is
+	// enabled (not just undecided).
+	cases := []struct {
+		opts    OptionsConfiguration
+		enabled bool
+	}{
+		{opts: OptionsConfiguration{URAccepted: 0, GlobalAnnEnabled: true}, enabled: true},
+		{opts: OptionsConfiguration{URAccepted: -1, GlobalAnnEnabled: true}, enabled: true},
+		{opts: OptionsConfiguration{URAccepted: 1, GlobalAnnEnabled: true}, enabled: true},
+		{opts: OptionsConfiguration{URAccepted: 0, GlobalAnnEnabled: false}, enabled: false},
+		{opts: OptionsConfiguration{URAccepted: -1, GlobalAnnEnabled: false}, enabled: false},
+		{opts: OptionsConfiguration{URAccepted: 1, GlobalAnnEnabled: false}, enabled: true},
+	}
+
+	for i, tc := range cases {
+		cfg := Configuration{Version: 28, Options: tc.opts}
+		migrations.apply(&cfg)
+		if cfg.Options.CREnabled != tc.enabled {
+			t.Errorf("%d: unexpected result, CREnabled: %v != %v", i, cfg.Options.CREnabled, tc.enabled)
+		}
+	}
+}
--- a/lib/config/optionsconfiguration.go
+++ b/lib/config/optionsconfiguration.go
@ -29,11 +29,11 @@ type OptionsConfiguration struct {
 	NATLeaseM               int      `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
 	NATRenewalM             int      `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
 	NATTimeoutS             int      `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
-	URAccepted              int      `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
-	URSeen                  int      `xml:"urSeen" json:"urSeen"`         // Report which the user has been prompted for.
-	URUniqueID              string   `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
-	URURL                   string   `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
-	URPostInsecurely        bool     `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
+	URAccepted              int      `xml:"urAccepted" json:"urAccepted"`                                    // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
+	URSeen                  int      `xml:"urSeen" json:"urSeen"`                                            // Report which the user has been prompted for.
+	URUniqueID              string   `xml:"urUniqueID" json:"urUniqueId"`                                    // Unique ID for reporting purposes, regenerated when UR is turned on.
+	URURL                   string   `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"` // usage reporting URL
+	URPostInsecurely        bool     `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"`        // For testing
 	URInitialDelayS         int      `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
 	RestartOnWakeup         bool     `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true" restart:"true"`
 	AutoUpgradeIntervalH    int      `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12" restart:"true"` // 0 for off
@ -52,6 +52,8 @@ type OptionsConfiguration struct {
 	DefaultFolderPath       string   `xml:"defaultFolderPath" json:"defaultFolderPath" default:"~"`
 	SetLowPriority          bool     `xml:"setLowPriority" json:"setLowPriority" default:"true"`
 	MaxConcurrentScans      int      `xml:"maxConcurrentScans" json:"maxConcurrentScans"`
+	CRURL                   string   `xml:"crashReportingURL" json:"crURL" default:"https://crash.syncthing.net/newcrash"` // crash reporting URL
+	CREnabled               bool     `xml:"crashReportingEnabled" json:"crashReportingEnabled" default:"true" restart:"true"`
 	StunKeepaliveStartS     int      `xml:"stunKeepaliveStartS" json:"stunKeepaliveStartS" default:"180"` // 0 for off
 	StunKeepaliveMinS       int      `xml:"stunKeepaliveMinS" json:"stunKeepaliveMinS" default:"20"`      // 0 for off
 	StunServers             []string `xml:"stunServer" json:"stunServers" default:"default"`
--- a/lib/config/testdata/overridenvalues.xml
+++ b/lib/config/testdata/overridenvalues.xml
@ -1,4 +1,4 @@
-<configuration version="14">
+<configuration version="29">
    <options>
        <listenAddress>tcp://:23000</listenAddress>
        <allowDelete>false</allowDelete>
@ -27,8 +27,10 @@
        <symlinksEnabled>false</symlinksEnabled>
        <limitBandwidthInLan>true</limitBandwidthInLan>
        <databaseBlockCacheMiB>42</databaseBlockCacheMiB>
-        <minHomeDiskFreePct>5.2</minHomeDiskFreePct>
+        <minHomeDiskFree unit="%">5.2</minHomeDiskFree>
        <urURL>https://localhost/newdata</urURL>
+        <urSeen>8</urSeen>
+        <urAccepted>4</urAccepted>
        <urInitialDelayS>800</urInitialDelayS>
        <urPostInsecurely>true</urPostInsecurely>
        <releasesURL>https://localhost/releases</releasesURL>
@ -36,8 +38,11 @@
        <tempIndexMinBlocks>100</tempIndexMinBlocks>
        <defaultFolderPath>/media/syncthing</defaultFolderPath>
        <setLowPriority>false</setLowPriority>
+        <crashReportingURL>https://localhost/newcrash</crashReportingURL>
+        <crashReportingEnabled>false</crashReportingEnabled>
        <stunKeepaliveStartS>9000</stunKeepaliveStartS>
        <stunKeepaliveMinS>900</stunKeepaliveMinS>
        <stunServer>foo</stunServer>
+        <unackedNotificationID>asdfasdf</unackedNotificationID>
    </options>
 </configuration>
--- a/lib/config/testdata/v29.xml
+++ b/lib/config/testdata/v29.xml
@ -0,0 +1,16 @@
+<configuration version="28">
+    <folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" fsWatcherEnabled="false" fsWatcherDelayS="10" autoNormalize="true">
+        <filesystemType>basic</filesystemType>
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+        <minDiskFree unit="%">1</minDiskFree>
+        <maxConflicts>-1</maxConflicts>
+        <fsync>true</fsync>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>tcp://a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>tcp://b</address>
+    </device>
+</configuration>
--- a/lib/ur/usage_report.go
+++ b/lib/ur/usage_report.go
@ -66,7 +66,8 @@ func New(cfg config.Wrapper, m model.Model, connectionsService connections.Servi
 // ReportData returns the data to be sent in a usage report with the currently
 // configured usage reporting version.
 func (s *Service) ReportData() map[string]interface{} {
-	return s.reportData(Version, false)
+	urVersion := s.cfg.Options().URAccepted
+	return s.reportData(urVersion, false)
 }

 // ReportDataPreview returns a preview of the data to be sent in a usage report