From 7fa699e1597ced4a516c2484806d3b1058f520d6 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Thu, 16 Apr 2020 10:09:33 +0200 Subject: [PATCH] build, lib/build: Build faster (#6538) This changes the build script to build all the things in one go invocation, instead of one invocation per cmd. This is a lot faster because it means more things get compiled concurrently. It's especially a lot faster when things *don't* need to be rebuilt, possibly because it only needs to build the dependency map and such once instead of once per binary. In order for this to work we need to be able to pass the same ldflags to all the binaries. This means we can't set the program name with an ldflag. When it needs to rebuild everything (go clean -cache): ( ./old-build -gocmd go1.14.2 build all 2> /dev/null; ) 65.82s user 11.28s system 574% cpu 13.409 total ( ./new-build -gocmd go1.14.2 build all 2> /dev/null; ) 63.26s user 7.12s system 1220% cpu 5.766 total On a subsequent run (nothing to build, just link the binaries): ( ./old-build -gocmd go1.14.2 build all 2> /dev/null; ) 26.58s user 7.53s system 582% cpu 5.853 total ( ./new-build -gocmd go1.14.2 build all 2> /dev/null; ) 18.66s user 2.45s system 1090% cpu 1.935 total --- build.go | 28 ++++++++++------------------ cmd/stdiscosrv/main.go | 2 +- cmd/strelaysrv/main.go | 5 +++-- lib/build/build.go | 20 ++++++++++++-------- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/build.go b/build.go index f03f66d22..f707dcd5c 100644 --- a/build.go +++ b/build.go @@ -25,7 +25,6 @@ import ( "os" "os/exec" "os/user" - "path" "path/filepath" "regexp" "runtime" @@ -405,12 +404,9 @@ func install(target target, tags []string) { defer shouldCleanupSyso(sysoPath) } - for _, pkg := range target.buildPkgs { - args := []string{"install", "-v"} - args = appendParameters(args, tags, pkg) - - runPrint(goCmd, args...) - } + args := []string{"install", "-v"} + args = appendParameters(args, tags, target.buildPkgs...) + runPrint(goCmd, args...) } func build(target target, tags []string) { @@ -438,15 +434,12 @@ func build(target target, tags []string) { defer shouldCleanupSyso(sysoPath) } - for _, pkg := range target.buildPkgs { - args := []string{"build", "-v"} - args = appendParameters(args, tags, pkg) - - runPrint(goCmd, args...) - } + args := []string{"build", "-v"} + args = appendParameters(args, tags, target.buildPkgs...) + runPrint(goCmd, args...) } -func appendParameters(args []string, tags []string, pkg string) []string { +func appendParameters(args []string, tags []string, pkgs ...string) []string { if pkgdir != "" { args = append(args, "-pkgdir", pkgdir) } @@ -462,7 +455,7 @@ func appendParameters(args []string, tags []string, pkg string) []string { if !debugBinary { // Regular binaries get version tagged and skip some debug symbols - args = append(args, "-ldflags", ldflags(path.Base(pkg))) + args = append(args, "-ldflags", ldflags()) } else { // -gcflags to disable optimizations and inlining. Skip -ldflags // because `Could not launch program: decoding dwarf section info at @@ -471,7 +464,7 @@ func appendParameters(args []string, tags []string, pkg string) []string { args = append(args, "-gcflags", "-N -l") } - return append(args, pkg) + return append(args, pkgs...) } func buildTar(target target) { @@ -765,14 +758,13 @@ func transifex() { runPrint(goCmd, "run", "../../../../script/transifexdl.go") } -func ldflags(program string) string { +func ldflags() string { b := new(strings.Builder) b.WriteString("-w") fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version=%s", version) fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp=%d", buildStamp()) fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User=%s", buildUser()) fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host=%s", buildHost()) - fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Program=%s", program) if v := os.Getenv("EXTRA_LDFLAGS"); v != "" { fmt.Fprintf(b, " %s", v) } diff --git a/cmd/stdiscosrv/main.go b/cmd/stdiscosrv/main.go index f87c668dc..74479951f 100644 --- a/cmd/stdiscosrv/main.go +++ b/cmd/stdiscosrv/main.go @@ -92,7 +92,7 @@ func main() { showVersion := flag.Bool("version", false, "Show version") flag.Parse() - log.Println(build.LongVersion) + log.Println(build.LongVersionFor("stdiscosrv")) if *showVersion { return } diff --git a/cmd/strelaysrv/main.go b/cmd/strelaysrv/main.go index 2300f1054..0d59c24fe 100644 --- a/cmd/strelaysrv/main.go +++ b/cmd/strelaysrv/main.go @@ -102,8 +102,9 @@ func main() { showVersion := flag.Bool("version", false, "Show version") flag.Parse() + longVer := build.LongVersionFor("strelaysrv") if *showVersion { - fmt.Println(build.LongVersion) + fmt.Println(longVer) return } @@ -135,7 +136,7 @@ func main() { } } - log.Println(build.LongVersion) + log.Println(longVer) maxDescriptors, err := osutil.MaximizeOpenFileLimit() if maxDescriptors > 0 { diff --git a/lib/build/build.go b/lib/build/build.go index 733fb9137..833f74b78 100644 --- a/lib/build/build.go +++ b/lib/build/build.go @@ -18,7 +18,6 @@ import ( var ( // Injected by build script - Program = "syncthing" Version = "unknown-dev" Host = "unknown" User = "unknown" @@ -72,11 +71,16 @@ func setBuildData() { stamp, _ := strconv.Atoi(Stamp) Date = time.Unix(int64(stamp), 0) - - date := Date.UTC().Format("2006-01-02 15:04:05 MST") - LongVersion = fmt.Sprintf(`%s %s "%s" (%s %s-%s) %s@%s %s`, Program, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, User, Host, date) - - if len(Tags) > 0 { - LongVersion = fmt.Sprintf("%s [%s]", LongVersion, strings.Join(Tags, ", ")) - } + LongVersion = LongVersionFor("syncthing") +} + +// LongVersionFor returns the long version string for the given program name. +func LongVersionFor(program string) string { + // This string and date format is essentially part of our external API. Never change it. + date := Date.UTC().Format("2006-01-02 15:04:05 MST") + v := fmt.Sprintf(`%s %s "%s" (%s %s-%s) %s@%s %s`, program, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, User, Host, date) + if len(Tags) > 0 { + v = fmt.Sprintf("%s [%s]", v, strings.Join(Tags, ", ")) + } + return v }