From 06365e56358afdd2d8d191221d75022534f8c2fd Mon Sep 17 00:00:00 2001 From: greatroar <61184462+greatroar@users.noreply.github.com> Date: Sun, 10 May 2020 11:44:34 +0200 Subject: [PATCH] cmd/strelaypoolsrv, lib/api: Factor out static asset serving (#6624) --- build.go | 2 +- cmd/strelaypoolsrv/main.go | 70 +++------------------- lib/api/api_statics.go | 76 +++-------------------- lib/{ => api}/auto/auto_test.go | 2 +- lib/{ => api}/auto/doc.go | 2 +- lib/assets/assets.go | 97 ++++++++++++++++++++++++++++++ lib/assets/assets_test.go | 103 ++++++++++++++++++++++++++++++++ 7 files changed, 219 insertions(+), 133 deletions(-) rename lib/{ => api}/auto/auto_test.go (93%) rename lib/{ => api}/auto/doc.go (80%) create mode 100644 lib/assets/assets.go create mode 100644 lib/assets/assets_test.go diff --git a/build.go b/build.go index f707dcd5c..0a1aa57d3 100644 --- a/build.go +++ b/build.go @@ -687,7 +687,7 @@ func listFiles(dir string) []string { func rebuildAssets() { os.Setenv("SOURCE_DATE_EPOCH", fmt.Sprint(buildStamp())) - runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto") + runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/api/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto") } func lazyRebuildAssets() { diff --git a/cmd/strelaypoolsrv/main.go b/cmd/strelaypoolsrv/main.go index c5dbb158f..32a3edbc3 100644 --- a/cmd/strelaypoolsrv/main.go +++ b/cmd/strelaypoolsrv/main.go @@ -12,7 +12,6 @@ import ( "io" "io/ioutil" "log" - "mime" "net" "net/http" "net/url" @@ -27,6 +26,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto" + "github.com/syncthing/syncthing/lib/assets" "github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/relay/client" "github.com/syncthing/syncthing/lib/sync" @@ -263,78 +263,22 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) { func handleAssets(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-cache, must-revalidate") - assets := auto.Assets() path := r.URL.Path[1:] if path == "" { path = "index.html" } - bs, ok := assets[path] + content, ok := auto.Assets()[path] if !ok { w.WriteHeader(http.StatusNotFound) return } - etag := fmt.Sprintf("%d", auto.Generated) - modified := time.Unix(auto.Generated, 0).UTC() - - w.Header().Set("Last-Modified", modified.Format(http.TimeFormat)) - w.Header().Set("Etag", etag) - - mtype := mimeTypeForFile(path) - if len(mtype) != 0 { - w.Header().Set("Content-Type", mtype) - } - - if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modified.Add(time.Second).After(t) { - w.WriteHeader(http.StatusNotModified) - return - } - - if match := r.Header.Get("If-None-Match"); match != "" { - if strings.Contains(match, etag) { - w.WriteHeader(http.StatusNotModified) - return - } - } - if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { - w.Header().Set("Content-Encoding", "gzip") - w.Header().Set("Content-Length", strconv.Itoa(len(bs))) - io.WriteString(w, bs) - } else { - // ungzip if browser not send gzip accepted header - var gr *gzip.Reader - gr, _ = gzip.NewReader(strings.NewReader(bs)) - io.Copy(w, gr) - gr.Close() - } -} - -func mimeTypeForFile(file string) string { - // We use a built in table of the common types since the system - // TypeByExtension might be unreliable. But if we don't know, we delegate - // to the system. - ext := filepath.Ext(file) - switch ext { - case ".htm", ".html": - return "text/html" - case ".css": - return "text/css" - case ".js": - return "application/javascript" - case ".json": - return "application/json" - case ".png": - return "image/png" - case ".ttf": - return "application/x-font-ttf" - case ".woff": - return "application/x-font-woff" - case ".svg": - return "image/svg+xml" - default: - return mime.TypeByExtension(ext) - } + assets.Serve(w, r, assets.Asset{ + ContentGz: content, + Filename: path, + Modified: time.Unix(auto.Generated, 0).UTC(), + }) } func handleRequest(w http.ResponseWriter, r *http.Request) { diff --git a/lib/api/api_statics.go b/lib/api/api_statics.go index 18a51c062..d0cdd6f74 100644 --- a/lib/api/api_statics.go +++ b/lib/api/api_statics.go @@ -7,18 +7,15 @@ package api import ( - "compress/gzip" "fmt" - "io" - "mime" "net/http" "os" "path/filepath" - "strconv" "strings" "time" - "github.com/syncthing/syncthing/lib/auto" + "github.com/syncthing/syncthing/lib/api/auto" + "github.com/syncthing/syncthing/lib/assets" "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/sync" ) @@ -111,7 +108,7 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) { if s.assetDir != "" { p := filepath.Join(s.assetDir, theme, filepath.FromSlash(file)) if _, err := os.Stat(p); err == nil { - mtype := s.mimeTypeForFile(file) + mtype := assets.MimeTypeForFile(file) if len(mtype) != 0 { w.Header().Set("Content-Type", mtype) } @@ -127,7 +124,7 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) { if s.assetDir != "" { p := filepath.Join(s.assetDir, config.DefaultTheme, filepath.FromSlash(file)) if _, err := os.Stat(p); err == nil { - mtype := s.mimeTypeForFile(file) + mtype := assets.MimeTypeForFile(file) if len(mtype) != 0 { w.Header().Set("Content-Type", mtype) } @@ -144,39 +141,11 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) { } } - etag := fmt.Sprintf("%d", modificationTime.Unix()) - w.Header().Set("Last-Modified", modificationTime.Format(http.TimeFormat)) - w.Header().Set("Etag", etag) - - if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil { - if modificationTime.Equal(t) || modificationTime.Before(t) { - w.WriteHeader(http.StatusNotModified) - return - } - } - - if match := r.Header.Get("If-None-Match"); match != "" { - if strings.Contains(match, etag) { - w.WriteHeader(http.StatusNotModified) - return - } - } - - mtype := s.mimeTypeForFile(file) - if len(mtype) != 0 { - w.Header().Set("Content-Type", mtype) - } - if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { - w.Header().Set("Content-Encoding", "gzip") - w.Header().Set("Content-Length", strconv.Itoa(len(bs))) - io.WriteString(w, bs) - } else { - // ungzip if browser not send gzip accepted header - var gr *gzip.Reader - gr, _ = gzip.NewReader(strings.NewReader(bs)) - io.Copy(w, gr) - gr.Close() - } + assets.Serve(w, r, assets.Asset{ + ContentGz: bs, + Filename: file, + Modified: modificationTime, + }) } func (s *staticsServer) serveThemes(w http.ResponseWriter, r *http.Request) { @@ -185,33 +154,6 @@ func (s *staticsServer) serveThemes(w http.ResponseWriter, r *http.Request) { }) } -func (s *staticsServer) mimeTypeForFile(file string) string { - // We use a built in table of the common types since the system - // TypeByExtension might be unreliable. But if we don't know, we delegate - // to the system. All our files are UTF-8. - ext := filepath.Ext(file) - switch ext { - case ".htm", ".html": - return "text/html; charset=utf-8" - case ".css": - return "text/css; charset=utf-8" - case ".js": - return "application/javascript; charset=utf-8" - case ".json": - return "application/json; charset=utf-8" - case ".png": - return "image/png" - case ".ttf": - return "application/x-font-ttf" - case ".woff": - return "application/x-font-woff" - case ".svg": - return "image/svg+xml; charset=utf-8" - default: - return mime.TypeByExtension(ext) - } -} - func (s *staticsServer) setTheme(theme string) { s.mut.Lock() s.theme = theme diff --git a/lib/auto/auto_test.go b/lib/api/auto/auto_test.go similarity index 93% rename from lib/auto/auto_test.go rename to lib/api/auto/auto_test.go index a33e3c7d3..73b7736c2 100644 --- a/lib/auto/auto_test.go +++ b/lib/api/auto/auto_test.go @@ -13,7 +13,7 @@ import ( "strings" "testing" - "github.com/syncthing/syncthing/lib/auto" + "github.com/syncthing/syncthing/lib/api/auto" ) func TestAssets(t *testing.T) { diff --git a/lib/auto/doc.go b/lib/api/auto/doc.go similarity index 80% rename from lib/auto/doc.go rename to lib/api/auto/doc.go index 80845e33a..f877a1109 100644 --- a/lib/auto/doc.go +++ b/lib/api/auto/doc.go @@ -4,7 +4,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -//go:generate go run ../../script/genassets.go -o gui.files.go ../../gui +//go:generate go run ../../../script/genassets.go -o gui.files.go ../../../gui // Package auto contains auto generated files for web assets. package auto diff --git a/lib/assets/assets.go b/lib/assets/assets.go new file mode 100644 index 000000000..38b3c275e --- /dev/null +++ b/lib/assets/assets.go @@ -0,0 +1,97 @@ +// Copyright (C) 2014-2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// Package assets hold utilities for serving static assets. +// +// The actual assets live in auto subpackages instead of here, +// because the set of assets varies per program. +package assets + +import ( + "compress/gzip" + "fmt" + "io" + "mime" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" +) + +// Asset is the type of arguments to Serve. +type Asset struct { + ContentGz string // gzipped contents of asset. + Filename string // Original filename, determines Content-Type. + Modified time.Time // Determines ETag and Last-Modified. +} + +// Serve writes a gzipped asset to w. +func Serve(w http.ResponseWriter, r *http.Request, asset Asset) { + header := w.Header() + + mtype := MimeTypeForFile(asset.Filename) + if mtype != "" { + header.Set("Content-Type", mtype) + } + + etag := fmt.Sprintf(`"%x"`, asset.Modified.Unix()) + header.Set("ETag", etag) + header.Set("Last-Modified", asset.Modified.Format(http.TimeFormat)) + + t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")) + if err == nil && !asset.Modified.After(t) { + w.WriteHeader(http.StatusNotModified) + return + } + + if r.Header.Get("If-None-Match") == etag { + w.WriteHeader(http.StatusNotModified) + return + } + + if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + header.Set("Content-Encoding", "gzip") + header.Set("Content-Length", strconv.Itoa(len(asset.ContentGz))) + io.WriteString(w, asset.ContentGz) + } else { + // gunzip for browsers that don't want gzip. + var gr *gzip.Reader + gr, _ = gzip.NewReader(strings.NewReader(asset.ContentGz)) + io.Copy(w, gr) + gr.Close() + } +} + +// MimeTypeForFile returns the appropriate MIME type for an asset, +// based on the filename. +// +// We use a built in table of the common types since the system +// TypeByExtension might be unreliable. But if we don't know, we delegate +// to the system. All our text files are in UTF-8. +func MimeTypeForFile(file string) string { + ext := filepath.Ext(file) + switch ext { + case ".htm", ".html": + return "text/html; charset=utf-8" + case ".css": + return "text/css; charset=utf-8" + case ".js": + return "application/javascript; charset=utf-8" + case ".json": + return "application/json; charset=utf-8" + case ".png": + return "image/png" + case ".ttf": + return "application/x-font-ttf" + case ".woff": + return "application/x-font-woff" + case ".svg": + return "image/svg+xml; charset=utf-8" + default: + return mime.TypeByExtension(ext) + } +} diff --git a/lib/assets/assets_test.go b/lib/assets/assets_test.go new file mode 100644 index 000000000..9e7caf52c --- /dev/null +++ b/lib/assets/assets_test.go @@ -0,0 +1,103 @@ +// Copyright (C) 2020 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package assets + +import ( + "bytes" + "compress/gzip" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func compress(s string) string { + var sb strings.Builder + gz := gzip.NewWriter(&sb) + + io.WriteString(gz, s) + gz.Close() + return sb.String() +} + +func decompress(p []byte) (out []byte) { + r, err := gzip.NewReader(bytes.NewBuffer(p)) + if err == nil { + out, err = ioutil.ReadAll(r) + } + if err != nil { + panic(err) + } + return out +} + +func TestServe(t *testing.T) { + indexHTML := `Hello, world!` + indexGz := compress(indexHTML) + + handler := func(w http.ResponseWriter, r *http.Request) { + Serve(w, r, Asset{ + ContentGz: indexGz, + Filename: r.URL.Path[1:], + Modified: time.Unix(0, 0), + }) + } + + for _, acceptGzip := range []bool{true, false} { + r := httptest.NewRequest("GET", "http://localhost/index.html", nil) + if acceptGzip { + r.Header.Set("accept-encoding", "gzip, deflate") + } + + w := httptest.NewRecorder() + handler(w, r) + res := w.Result() + + if res.StatusCode != http.StatusOK { + t.Fatalf("wanted OK, got status %d", res.StatusCode) + } + if ctype := res.Header.Get("Content-Type"); ctype != "text/html; charset=utf-8" { + t.Errorf("unexpected Content-Type %q", ctype) + } + // ETags must be quoted ASCII strings: + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag + if etag := res.Header.Get("ETag"); etag != `"0"` { + t.Errorf("unexpected ETag %q", etag) + } + + body, _ := ioutil.ReadAll(res.Body) + if acceptGzip { + body = decompress(body) + } + if string(body) != indexHTML { + t.Fatalf("unexpected content %q", body) + } + } + + r := httptest.NewRequest("GET", "http://localhost/index.html", nil) + r.Header.Set("if-none-match", `"0"`) + w := httptest.NewRecorder() + handler(w, r) + res := w.Result() + + if res.StatusCode != http.StatusNotModified { + t.Fatalf("wanted NotModified, got status %d", res.StatusCode) + } + + r = httptest.NewRequest("GET", "http://localhost/index.html", nil) + r.Header.Set("if-modified-since", time.Now().Format(http.TimeFormat)) + w = httptest.NewRecorder() + handler(w, r) + res = w.Result() + + if res.StatusCode != http.StatusNotModified { + t.Fatalf("wanted NotModified, got status %d", res.StatusCode) + } +}