Martchus
/
syncthing
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

meta: Move metadata checks into meta directory, make them tests 

This moves a few things from script/ to a new directory meta/, and makes
them real Go tests. These are the authors, copyright, metalint and gofmt
checks. That means that they can now be run by

go test -v ./meta

and optionally filtered by the usual -run thing to go test. Also -short
will cut down on the metalint stuff and exclude the authors check (which
is slow because it runs git lots of times).

Mainly this makes everything easier on things like build servers where
we can now just run tests instead of do a bunch of scripting.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4252
This commit is contained in:
Jakob Borg 2017-07-07 20:43:26 +00:00
parent 5a38e0ba3f
commit 200a7fc844
6 changed files with 208 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
build.go
View File

@ -170,41 +170,6 @@ var targets = map[string]target{
},
}
var (
// fast linters complete in a fraction of a second and might as well be
// run always as part of the build
fastLinters = []string{
"deadcode",
"golint",
"ineffassign",
"vet",
}
// slow linters take several seconds and are run only as part of the
// "metalint" command.
slowLinters = []string{
"gosimple",
"staticcheck",
"structcheck",
"unused",
"varcheck",
}
// Which parts of the tree to lint
lintDirs = []string{".", "./lib/...", "./cmd/..."}
// Messages to ignore
lintExcludes = []string{
".pb.go",
"should have comment",
"protocol.Vector composite literal uses unkeyed fields",
"cli.Requires composite literal uses unkeyed fields",
"Use DialContext instead", // Go 1.7
"os.SEEK_SET is deprecated", // Go 1.7
"SA4017", // staticcheck "is a pure function but its return value is ignored"
}
)
func init() {
// The "syncthing" target includes a few more files found in the "etc"
// and "extra" dirs.
@ -285,7 +250,7 @@ func runCommand(cmd string, target target) {
tags = []string{"noupgrade"}
}
install(target, tags)
metalint(fastLinters, lintDirs)
metalintShort()
case "build":
var tags []string
@ -293,7 +258,7 @@ func runCommand(cmd string, target target) {
tags = []string{"noupgrade"}
}
build(target, tags)
metalint(fastLinters, lintDirs)
metalintShort()
case "test":
test("./lib/...", "./cmd/...")
@ -329,14 +294,13 @@ func runCommand(cmd string, target target) {
clean()
case "vet":
metalint(fastLinters, lintDirs)
metalintShort()
case "lint":
metalint(fastLinters, lintDirs)
metalintShort()
case "metalint":
metalint(fastLinters, lintDirs)
metalint(slowLinters, lintDirs)
metalint()
case "version":
fmt.Println(getVersion())
@ -1055,59 +1019,12 @@ func macosCodesign(file string) {
}
}
func metalint(linters []string, dirs []string) {
ok := true
if isGometalinterInstalled() {
if !gometalinter(linters, dirs, lintExcludes...) {
ok = false
}
}
if !ok {
log.Fatal("Build succeeded, but there were lint warnings")
}
func metalint() {
lazyRebuildAssets()
runPrint("go", "test", "-run", "Metalint", "./meta")
}
func isGometalinterInstalled() bool {
if _, err := runError("gometalinter", "--disable-all"); err != nil {
log.Println("gometalinter is not installed")
return false
}
return true
}
func gometalinter(linters []string, dirs []string, excludes ...string) bool {
params := []string{"--disable-all", "--concurrency=2", "--deadline=300s"}
for _, linter := range linters {
params = append(params, "--enable="+linter)
}
for _, exclude := range excludes {
params = append(params, "--exclude="+exclude)
}
for _, dir := range dirs {
params = append(params, dir)
}
bs, _ := runError("gometalinter", params...)
nerr := 0
lines := make(map[string]struct{})
for _, line := range strings.Split(string(bs), "\n") {
if line == "" {
continue
}
if _, ok := lines[line]; ok {
continue
}
log.Println(line)
if strings.Contains(line, "executable file not found") {
log.Println(` - Try "go run build.go setup" to install missing tools`)
}
lines[line] = struct{}{}
nerr++
}
return nerr == 0
func metalintShort() {
lazyRebuildAssets()
runPrint("go", "test", "-short", "-run", "Metalint", "./meta")
}

3
meta/README.txt Normal file
View File

@ -0,0 +1,3 @@
The files in this directory contain metadata tests - that is, tests on the
shape and colour of the code in the rest of the repository. This code is not
compiled into the final product.

52
script/check-authors.go → meta/authors_test.go
View File

@ -4,19 +4,16 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build ignore
// Checks for authors that are not mentioned in AUTHORS
package main
package meta
import (
"bytes"
"io/ioutil"
"log"
"os"
"os/exec"
"regexp"
"strings"
"testing"
)
// list of commits that we don't include in our checks; because they are
@ -37,34 +34,36 @@ var excludeCommits = stringSetFromStrings([]string{
"bcc5d7c00f52552303b463d43a636f27b7f7e19b",
})
func init() {
log.SetOutput(os.Stdout)
log.SetFlags(0)
}
func TestCheckAuthors(t *testing.T) {
if testing.Short() {
t.Skip("skipping slow test")
}
func main() {
actual := actualAuthorEmails("cmd/", "lib/", "gui/", "test/", "script/")
listed := listedAuthorEmails()
actual, hashes := actualAuthorEmails(t, ".", "../cmd/", "../lib/", "../gui/", "../test/", "../script/")
listed := listedAuthorEmails(t)
missing := actual.except(listed)
if len(missing) > 0 {
log.Println("Missing authors:")
for author := range missing {
log.Println(" ", author)
for author := range missing {
t.Logf("Missing author: %s", author)
for _, hash := range hashes[author] {
t.Logf(" in hash: %s", hash)
}
os.Exit(1)
}
if len(missing) > 0 {
t.Errorf("Missing %d author(s)", len(missing))
}
}
// actualAuthorEmails returns the set of author emails found in the actual git
// commit log, except those in excluded commits.
func actualAuthorEmails(paths ...string) stringSet {
func actualAuthorEmails(t *testing.T, paths ...string) (stringSet, map[string][]string) {
args := append([]string{"log", "--format=%H %ae"}, paths...)
cmd := exec.Command("git", args...)
bs, err := cmd.Output()
if err != nil {
log.Fatal("authorEmails:", err)
t.Fatal("authorEmails:", err)
}
hashes := make(map[string][]string)
authors := newStringSet()
for _, line := range bytes.Split(bs, []byte{'\n'}) {
fields := strings.Fields(string(line))
@ -77,21 +76,22 @@ func actualAuthorEmails(paths ...string) stringSet {
continue
}
if strings.Contains(strings.ToLower(body(hash)), "skip-check: authors") {
if strings.Contains(strings.ToLower(body(t, hash)), "skip-check: authors") {
continue
}
authors.add(author)
hashes[author] = append(hashes[author], hash)
}
return authors
return authors, hashes
}
// listedAuthorEmails returns the set of author emails mentioned in AUTHORS
func listedAuthorEmails() stringSet {
bs, err := ioutil.ReadFile("AUTHORS")
func listedAuthorEmails(t *testing.T) stringSet {
bs, err := ioutil.ReadFile("../AUTHORS")
if err != nil {
log.Fatal("listedAuthorEmails:", err)
t.Fatal("listedAuthorEmails:", err)
}
emailRe := regexp.MustCompile(`<([^>]+)>`)
@ -104,11 +104,11 @@ func listedAuthorEmails() stringSet {
return authors
}
func body(hash string) string {
func body(t *testing.T, hash string) string {
cmd := exec.Command("git", "show", "--pretty=format:%b", "-s", hash)
bs, err := cmd.Output()
if err != nil {
log.Fatal("body:", err)
t.Fatal("body:", err)
}
return string(bs)
}

21
script/check-copyright.go → meta/copyright_test.go
View File

@ -4,26 +4,27 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// +build ignore
// Checks for files missing copyright notice
package main
package meta
import (
"bufio"
"flag"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"testing"
)
// File extensions to check
var checkExts = map[string]bool{
var copyrightCheckExts = map[string]bool{
".go": true,
}
// Directories to search
var copyrightCheckDirs = []string{".", "../cmd", "../lib", "../test", "../script"}
// Valid copyright headers, searched for in the top five lines in each file.
var copyrightRegexps = []string{
`Copyright`,
@ -34,13 +35,11 @@ var copyrightRegexps = []string{
var copyrightRe = regexp.MustCompile(strings.Join(copyrightRegexps, "|"))
func main() {
flag.Parse()
for _, dir := range flag.Args() {
func TestCheckCopyright(t *testing.T) {
for _, dir := range copyrightCheckDirs {
err := filepath.Walk(dir, checkCopyright)
if err != nil {
fmt.Println(err)
os.Exit(1)
t.Error(err)
}
}
}
@ -52,7 +51,7 @@ func checkCopyright(path string, info os.FileInfo, err error) error {
if !info.Mode().IsRegular() {
return nil
}
if !checkExts[filepath.Ext(path)] {
if !copyrightCheckExts[filepath.Ext(path)] {
return nil
}

45
meta/gofmt_test.go Normal file
View File

@ -0,0 +1,45 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Checks for authors that are not mentioned in AUTHORS
package meta
import (
"os"
"os/exec"
"path/filepath"
"testing"
)
var gofmtCheckDirs = []string{".", "../cmd", "../lib", "../test", "../script"}
func TestCheckGoFmt(t *testing.T) {
for _, dir := range gofmtCheckDirs {
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if path == ".git" {
return filepath.SkipDir
}
if filepath.Ext(path) != ".go" {
return nil
}
cmd := exec.Command("gofmt", "-s", "-d", path)
bs, err := cmd.CombinedOutput()
if err != nil {
return err
}
if len(bs) != 0 {
t.Errorf("File %s is not formatted correctly:\n\n%s", path, string(bs))
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
}

113
meta/metalint_test.go Normal file
View File

@ -0,0 +1,113 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package meta
import (
"bytes"
"log"
"os/exec"
"strings"
"testing"
)
var (
// fast linters complete in a fraction of a second and might as well be
// run always as part of the build
fastLinters = []string{
"deadcode",
"golint",
"ineffassign",
"vet",
}
// slow linters take several seconds and are run only as part of the
// "metalint" command.
slowLinters = []string{
"gosimple",
"staticcheck",
"structcheck",
"unused",
"varcheck",
}
// Which parts of the tree to lint
lintDirs = []string{".", "../script/...", "../lib/...", "../cmd/..."}
// Messages to ignore
lintExcludes = []string{
".pb.go",
"should have comment",
"protocol.Vector composite literal uses unkeyed fields",
"cli.Requires composite literal uses unkeyed fields",
"Use DialContext instead", // Go 1.7
"os.SEEK_SET is deprecated", // Go 1.7
"SA4017", // staticcheck "is a pure function but its return value is ignored"
}
)
func TestCheckMetalint(t *testing.T) {
if !isGometalinterInstalled() {
return
}
gometalinter(t, lintDirs, lintExcludes...)
}
func isGometalinterInstalled() bool {
if _, err := runError("gometalinter", "--disable-all"); err != nil {
log.Println("gometalinter is not installed")
return false
}
return true
}
func gometalinter(t *testing.T, dirs []string, excludes ...string) bool {
params := []string{"--disable-all", "--concurrency=2", "--deadline=300s"}
for _, linter := range fastLinters {
params = append(params, "--enable="+linter)
}
if !testing.Short() {
for _, linter := range slowLinters {
params = append(params, "--enable="+linter)
}
}
for _, exclude := range excludes {
params = append(params, "--exclude="+exclude)
}
params = append(params, dirs...)
bs, _ := runError("gometalinter", params...)
nerr := 0
lines := make(map[string]struct{})
for _, line := range strings.Split(string(bs), "\n") {
if line == "" {
continue
}
if _, ok := lines[line]; ok {
continue
}
log.Println(line)
if strings.Contains(line, "executable file not found") {
log.Println(` - Try "go run build.go setup" to install missing tools`)
}
lines[line] = struct{}{}
nerr++
}
return nerr == 0
}
func runError(cmd string, args ...string) ([]byte, error) {
ecmd := exec.Command(cmd, args...)
bs, err := ecmd.CombinedOutput()
return bytes.TrimSpace(bs), err
}