build, lib/build: Build faster (#6538)

This changes the build script to build all the things in one go
invocation, instead of one invocation per cmd. This is a lot faster
because it means more things get compiled concurrently. It's especially
a lot faster when things *don't* need to be rebuilt, possibly because it
only needs to build the dependency map and such once instead of once per
binary.

In order for this to work we need to be able to pass the same ldflags to
all the binaries. This means we can't set the program name with an
ldflag.

When it needs to rebuild everything (go clean -cache):

    ( ./old-build -gocmd go1.14.2 build all 2> /dev/null; )  65.82s user 11.28s system 574% cpu 13.409 total
    ( ./new-build -gocmd go1.14.2 build all 2> /dev/null; )  63.26s user 7.12s system 1220% cpu 5.766 total

On a subsequent run (nothing to build, just link the binaries):

    ( ./old-build -gocmd go1.14.2 build all 2> /dev/null; )  26.58s user 7.53s system 582% cpu 5.853 total
    ( ./new-build -gocmd go1.14.2 build all 2> /dev/null; )  18.66s user 2.45s system 1090% cpu 1.935 total

build.go

@@ -25,7 +25,6 @@ import (
 	"os"
 	"os/exec"
 	"os/user"
-	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -405,12 +404,9 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	for _, pkg := range target.buildPkgs {
-		args := []string{"install", "-v"}
-		args = appendParameters(args, tags, pkg)
-
-		runPrint(goCmd, args...)
-	}
+	args := []string{"install", "-v"}
+	args = appendParameters(args, tags, target.buildPkgs...)
+	runPrint(goCmd, args...)
 }
 
 func build(target target, tags []string) {
@@ -438,15 +434,12 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	for _, pkg := range target.buildPkgs {
-		args := []string{"build", "-v"}
-		args = appendParameters(args, tags, pkg)
-
-		runPrint(goCmd, args...)
-	}
+	args := []string{"build", "-v"}
+	args = appendParameters(args, tags, target.buildPkgs...)
+	runPrint(goCmd, args...)
 }
 
-func appendParameters(args []string, tags []string, pkg string) []string {
+func appendParameters(args []string, tags []string, pkgs ...string) []string {
 	if pkgdir != "" {
 		args = append(args, "-pkgdir", pkgdir)
 	}
@@ -462,7 +455,7 @@ func appendParameters(args []string, tags []string, pkg string) []string {
 
 	if !debugBinary {
 		// Regular binaries get version tagged and skip some debug symbols
-		args = append(args, "-ldflags", ldflags(path.Base(pkg)))
+		args = append(args, "-ldflags", ldflags())
 	} else {
 		// -gcflags to disable optimizations and inlining. Skip -ldflags
 		// because `Could not launch program: decoding dwarf section info at
@@ -471,7 +464,7 @@ func appendParameters(args []string, tags []string, pkg string) []string {
 		args = append(args, "-gcflags", "-N -l")
 	}
 
-	return append(args, pkg)
+	return append(args, pkgs...)
 }
 
 func buildTar(target target) {
@@ -765,14 +758,13 @@ func transifex() {
 	runPrint(goCmd, "run", "../../../../script/transifexdl.go")
 }
 
-func ldflags(program string) string {
+func ldflags() string {
 	b := new(strings.Builder)
 	b.WriteString("-w")
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version=%s", version)
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp=%d", buildStamp())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User=%s", buildUser())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host=%s", buildHost())
-	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Program=%s", program)
 	if v := os.Getenv("EXTRA_LDFLAGS"); v != "" {
 		fmt.Fprintf(b, " %s", v)
 	}

cmd/stdiscosrv/main.go

@@ -92,7 +92,7 @@ func main() {
 	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
-	log.Println(build.LongVersion)
+	log.Println(build.LongVersionFor("stdiscosrv"))
 	if *showVersion {
 		return
 	}

cmd/strelaysrv/main.go

@@ -102,8 +102,9 @@ func main() {
 	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
+	longVer := build.LongVersionFor("strelaysrv")
 	if *showVersion {
-		fmt.Println(build.LongVersion)
+		fmt.Println(longVer)
 		return
 	}
 
@@ -135,7 +136,7 @@ func main() {
 		}
 	}
 
-	log.Println(build.LongVersion)
+	log.Println(longVer)
 
 	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
 	if maxDescriptors > 0 {

lib/build/build.go

@@ -18,7 +18,6 @@ import (
 
 var (
 	// Injected by build script
-	Program = "syncthing"
 	Version = "unknown-dev"
 	Host    = "unknown"
 	User    = "unknown"
@@ -72,11 +71,16 @@ func setBuildData() {
 
 	stamp, _ := strconv.Atoi(Stamp)
 	Date = time.Unix(int64(stamp), 0)
-
-	date := Date.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`%s %s "%s" (%s %s-%s) %s@%s %s`, Program, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, User, Host, date)
-
-	if len(Tags) > 0 {
-		LongVersion = fmt.Sprintf("%s [%s]", LongVersion, strings.Join(Tags, ", "))
-	}
+	LongVersion = LongVersionFor("syncthing")
+}
+
+// LongVersionFor returns the long version string for the given program name.
+func LongVersionFor(program string) string {
+	// This string and date format is essentially part of our external API. Never change it.
+	date := Date.UTC().Format("2006-01-02 15:04:05 MST")
+	v := fmt.Sprintf(`%s %s "%s" (%s %s-%s) %s@%s %s`, program, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, User, Host, date)
+	if len(Tags) > 0 {
+		v = fmt.Sprintf("%s [%s]", v, strings.Join(Tags, ", "))
+	}
+	return v
 }