diff --git a/.github/workflows/reproducibility.yml b/.github/workflows/reproducibility.yml index 3363397..d519870 100644 --- a/.github/workflows/reproducibility.yml +++ b/.github/workflows/reproducibility.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # goreleaser and the ldflags below read the commit date + # goreleaser and the Go version stamp read the git history fetch-depth: 0 persist-credentials: false - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -30,14 +30,12 @@ jobs: - name: Rebuild from source and compare digests run: | - VERSION=$(jq -r .version dist/metadata.json) - DATE=$(git log -1 --format=%cd --date=format-local:'%Y-%m-%dT%H:%M:%SZ') cp -a . /tmp/src-elsewhere cd /tmp/src-elsewhere CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOAMD64=v1 \ go build -trimpath \ - -ldflags="-s -w -buildid= -X main.version=$VERSION -X main.date=$DATE" \ - -o /tmp/plain-build ./publiccode-parser/publiccode_parser.go + -ldflags="-s -w -buildid=" \ + -o /tmp/plain-build ./publiccode-parser sha256sum /tmp/goreleaser-build /tmp/plain-build cmp /tmp/goreleaser-build /tmp/plain-build \ && echo "Reproducible build confirmed" diff --git a/.gitignore b/.gitignore index 50e58d2..1249387 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ publiccode-parser/publiccode-parser +dist/ .vscode .history .DS_Store diff --git a/.golangci.yaml b/.golangci.yaml index d4ec8f0..ab1234f 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -67,6 +67,9 @@ linters: # No point in wrapping these - func encoding/json.Marshal(v any) - func encoding/json.UnmarshalJSON(v any) + # A RoundTripper must return the transport error as is: http.Client + # and its callers type assert on it instead of unwrapping it + - func (net/http.RoundTripper).RoundTrip( # Defaults - .Errorf( diff --git a/.goreleaser.yml b/.goreleaser.yml index fbbd013..60e737f 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -11,11 +11,11 @@ builds: - id: publiccode-parser binary: publiccode-parser - main: ./publiccode-parser/publiccode_parser.go + main: ./publiccode-parser flags: - -trimpath ldflags: - - -s -w -buildid= -X main.version={{.Version}} -X main.date={{.CommitDate}} + - -s -w -buildid= env: - CGO_ENABLED=0 - SOURCE_DATE_EPOCH={{.CommitTimestamp}} diff --git a/internal/safehttp.go b/internal/safehttp.go index be4e135..a708941 100644 --- a/internal/safehttp.go +++ b/internal/safehttp.go @@ -107,7 +107,7 @@ type safeTransport struct { func (t *safeTransport) RoundTrip(req *http.Request) (*http.Response, error) { resp, err := t.base.RoundTrip(req) if err != nil { - return nil, err //nolint:wrapcheck // http.Client inspects the transport error; keep it intact + return nil, err } // Reject early when the server advertises an over-limit Content-Length. @@ -122,11 +122,52 @@ func (t *safeTransport) RoundTrip(req *http.Request) (*http.Response, error) { return resp, nil } +// userAgentTransport sets a User-Agent on the requests that don't carry one, +// since WAFs often block Go's default "Go-http-client/". +type userAgentTransport struct { + base http.RoundTripper + userAgent string +} + +func (t *userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if _, set := req.Header["User-Agent"]; !set && req.Header != nil { + // A RoundTripper must not modify the request it is given, hence the clone. + req = req.Clone(req.Context()) + req.Header.Set("User-Agent", t.userAgent) + } + + return t.base.RoundTrip(req) +} + +// Option configures the client built by [SafeHTTPClient]. +type Option func(*options) + +type options struct { + userAgent string +} + +// WithUserAgent sets the User-Agent sent on every request that doesn't already +// carry one, in place of [UserAgent]. An empty value keeps that default. +func WithUserAgent(userAgent string) Option { + return func(o *options) { + o.userAgent = userAgent + } +} + // SafeHTTPClient builds an *http.Client hardened against SSRF and unbounded // downloads. When allowPrivate is true the SSRF address filtering is disabled // (used for trusted input and tests that target loopback servers); the response // size limit is always enforced. -func SafeHTTPClient(timeout time.Duration, allowPrivate bool) *http.Client { +func SafeHTTPClient(timeout time.Duration, allowPrivate bool, opts ...Option) *http.Client { + o := options{userAgent: UserAgent()} + for _, opt := range opts { + opt(&o) + } + + if o.userAgent == "" { + o.userAgent = UserAgent() + } + dialer := &net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, @@ -146,7 +187,10 @@ func SafeHTTPClient(timeout time.Duration, allowPrivate bool) *http.Client { } return &http.Client{ - Timeout: timeout, - Transport: &safeTransport{base: transport, max: MaxResponseBytes}, + Timeout: timeout, + Transport: &userAgentTransport{ + base: &safeTransport{base: transport, max: MaxResponseBytes}, + userAgent: o.userAgent, + }, } } diff --git a/internal/safehttp_test.go b/internal/safehttp_test.go index 5f5e7b3..ff8137a 100644 --- a/internal/safehttp_test.go +++ b/internal/safehttp_test.go @@ -134,3 +134,110 @@ func TestResponseUnderLimitIsReadFully(t *testing.T) { t.Errorf("body mismatch: got %q, want %q", got, body) } } + +func TestUserAgentTransportSetsTheDefault(t *testing.T) { + const userAgent = "libpubliccode/1.2.3 (+https://example.org)" + + var got string + transport := &userAgentTransport{ + base: roundTripFunc(func(r *http.Request) (*http.Response, error) { + got = r.Header.Get("User-Agent") + + return newResponse("", 0), nil + }), + userAgent: userAgent, + } + + req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil) + if _, err := transport.RoundTrip(req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if got != userAgent { + t.Errorf("User-Agent sent: got %q, want %q", got, userAgent) + } + + // A RoundTripper must not modify the request it is handed. + if _, set := req.Header["User-Agent"]; set { + t.Errorf("the original request was modified: %q", req.Header.Get("User-Agent")) + } +} + +func TestUserAgentTransportKeepsTheCallerHeader(t *testing.T) { + cases := map[string]string{ + "caller sets its own": "harvester/2.0", + "caller clears it": "", + } + + for name, want := range cases { + t.Run(name, func(t *testing.T) { + var got string + var seen bool + + transport := &userAgentTransport{ + base: roundTripFunc(func(r *http.Request) (*http.Response, error) { + got, seen = r.Header.Get("User-Agent"), true + + return newResponse("", 0), nil + }), + userAgent: "libpubliccode/1.2.3", + } + + req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil) + req.Header["User-Agent"] = []string{want} + + if _, err := transport.RoundTrip(req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !seen { + t.Fatal("the base transport was not called") + } + if got != want { + t.Errorf("User-Agent sent: got %q, want %q", got, want) + } + }) + } +} + +// TestSafeHTTPClientSendsUserAgent checks the User-Agent all the way down to the +// wire, and that the library's own is sent when none is given. +func TestSafeHTTPClientSendsUserAgent(t *testing.T) { + const userAgent = "libpubliccode/1.2.3 (+https://example.org)" + + // The server echoes back what it received, so there is nothing shared + // between the handler and the test. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, r.UserAgent()) + })) + defer srv.Close() + + got, err := getBody(SafeHTTPClient(5*time.Second, true, WithUserAgent(userAgent)), srv.URL) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != userAgent { + t.Errorf("User-Agent received by the server: got %q, want %q", got, userAgent) + } + + if got, err = getBody(SafeHTTPClient(5*time.Second, true), srv.URL); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != UserAgent() { + t.Errorf("User-Agent received by the server: got %q, want the default %q", got, UserAgent()) + } +} + +// getBody GETs url and returns the response body as a string. +func getBody(client *http.Client, url string) (string, error) { + resp, err := client.Get(url) + if err != nil { + return "", err + } + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + + return string(body), err +} diff --git a/internal/useragent.go b/internal/useragent.go new file mode 100644 index 0000000..cacc25c --- /dev/null +++ b/internal/useragent.go @@ -0,0 +1,97 @@ +package netutil + +import ( + "fmt" + "runtime/debug" + "strings" +) + +const ( + // productName is the token identifying this software in the User-Agent. + productName = "libpubliccode" + + // projectURL is part of the User-Agent so that whoever finds these requests + // in their logs can tell what made them, and allow them explicitly instead + // of having to allow every Go program. + projectURL = "https://github.com/publiccodeyml/libpubliccode" + + // modulePath is this module's import path. The version is looked up under it + // in the build information: the parser is the main module when its own CLI + // runs, and a dependency when the library is embedded in another program. + modulePath = "github.com/publiccodeyml/libpubliccode/v5" + + // unknownVersion stands in when the build information carries no usable + // version, as happens with "go run", "go test" and unstamped builds. + unknownVersion = "devel" +) + +// userAgent is resolved once: the build information doesn't change at runtime. +var userAgent = buildUserAgent(buildInfo()) + +// UserAgent returns the User-Agent naming this library and its version, e.g. +// +// libpubliccode/5.4.3 (+https://github.com/publiccodeyml/libpubliccode) +// +// The version comes from the build information of the running binary, and is +// "devel" when there is none to be found. +func UserAgent() string { + return userAgent +} + +// userAgentForVersion formats the User-Agent for a Go module version. +func userAgentForVersion(version string) string { + return fmt.Sprintf("%s/%s (+%s)", productName, normalizeVersion(version), projectURL) +} + +// buildInfo returns the build information of the running binary, or nil when it +// is unavailable. +func buildInfo() *debug.BuildInfo { + info, _ := debug.ReadBuildInfo() + + return info +} + +// buildUserAgent formats the User-Agent for the version of this module recorded +// in info. +func buildUserAgent(info *debug.BuildInfo) string { + return userAgentForVersion(moduleVersion(info)) +} + +// moduleVersion digs the version of this module out of info. It answers an empty +// string when info records none, which is the case for a binary built from a +// list of files (as the released CLI is) and when info itself is nil. +func moduleVersion(info *debug.BuildInfo) string { + if info == nil { + return "" + } + + // The main module is this one when one of its own commands runs: its path is + // the module path for the module itself, and below it for a command. + if info.Main.Path == modulePath || strings.HasPrefix(info.Main.Path, modulePath+"/") { + return info.Main.Version + } + + // The parser is a dependency when the library is embedded in another program. + for _, dep := range info.Deps { + if dep != nil && dep.Path == modulePath { + return dep.Version + } + } + + return "" +} + +// normalizeVersion turns a Go module version into what goes in the User-Agent: +// a bare version without the "v" prefix, or unknownVersion when there is +// nothing usable. A pseudo-version is kept as it is, since it identifies the +// commit. +func normalizeVersion(version string) string { + version = strings.TrimPrefix(version, "v") + + // An unstamped main module reports "(devel)". + if version == "" || strings.HasPrefix(version, "(") { + return unknownVersion + } + + return version +} diff --git a/internal/useragent_test.go b/internal/useragent_test.go new file mode 100644 index 0000000..44de038 --- /dev/null +++ b/internal/useragent_test.go @@ -0,0 +1,101 @@ +package netutil + +import ( + "runtime/debug" + "strings" + "testing" +) + +func TestBuildUserAgent(t *testing.T) { + cases := []struct { + name string + info *debug.BuildInfo + want string + }{ + { + name: "no build information", + info: nil, + want: "libpubliccode/devel (+https://github.com/publiccodeyml/libpubliccode)", + }, + { + name: "released CLI: this module is the main one", + info: &debug.BuildInfo{Main: debug.Module{Path: modulePath, Version: "v5.4.3"}}, + want: "libpubliccode/5.4.3 (+https://github.com/publiccodeyml/libpubliccode)", + }, + { + name: "unstamped build of the CLI", + info: &debug.BuildInfo{Main: debug.Module{Path: modulePath, Version: "(devel)"}}, + want: "libpubliccode/devel (+https://github.com/publiccodeyml/libpubliccode)", + }, + { + name: "embedded as a library: this module is a dependency", + info: &debug.BuildInfo{ + Main: debug.Module{Path: "example.org/harvester", Version: "v1.0.0"}, + Deps: []*debug.Module{ + {Path: "example.org/other", Version: "v0.1.0"}, + {Path: modulePath, Version: "v5.4.3"}, + }, + }, + want: "libpubliccode/5.4.3 (+https://github.com/publiccodeyml/libpubliccode)", + }, + { + name: "pseudo-version identifies the commit and is kept", + info: &debug.BuildInfo{ + Main: debug.Module{Path: "example.org/harvester", Version: "v1.0.0"}, + Deps: []*debug.Module{{Path: modulePath, Version: "v5.4.4-0.20260316100201-5dd490bc4896"}}, + }, + want: "libpubliccode/5.4.4-0.20260316100201-5dd490bc4896 " + + "(+https://github.com/publiccodeyml/libpubliccode)", + }, + { + name: "CLI installed with \"go install\": the main module is the command", + info: &debug.BuildInfo{Main: debug.Module{Path: modulePath + "/publiccode-parser", Version: "v5.4.3"}}, + want: "libpubliccode/5.4.3 (+https://github.com/publiccodeyml/libpubliccode)", + }, + { + name: "release build of the CLI: no module version in the build information", + info: &debug.BuildInfo{Main: debug.Module{Path: "command-line-arguments"}}, + want: "libpubliccode/devel (+https://github.com/publiccodeyml/libpubliccode)", + }, + { + name: "this module is nowhere to be found", + info: &debug.BuildInfo{Main: debug.Module{Path: "example.org/harvester", Version: "v1.0.0"}}, + want: "libpubliccode/devel (+https://github.com/publiccodeyml/libpubliccode)", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := buildUserAgent(tc.info); got != tc.want { + t.Errorf("buildUserAgent() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestUserAgentForVersionFormatsModuleVersions(t *testing.T) { + cases := map[string]string{ + "5.4.3": "libpubliccode/5.4.3 (+https://github.com/publiccodeyml/libpubliccode)", + "v5.4.3": "libpubliccode/5.4.3 (+https://github.com/publiccodeyml/libpubliccode)", + "devel": "libpubliccode/devel (+https://github.com/publiccodeyml/libpubliccode)", + "(devel)": "libpubliccode/devel (+https://github.com/publiccodeyml/libpubliccode)", + "": "libpubliccode/devel (+https://github.com/publiccodeyml/libpubliccode)", + } + + for version, want := range cases { + if got := userAgentForVersion(version); got != want { + t.Errorf("userAgentForVersion(%q) = %q, want %q", version, got, want) + } + } +} + +func TestUserAgentNamesThisProject(t *testing.T) { + got := UserAgent() + + if !strings.HasPrefix(got, productName+"/") { + t.Errorf("UserAgent() = %q, want it to start with %q", got, productName+"/") + } + if !strings.Contains(got, projectURL) { + t.Errorf("UserAgent() = %q, want it to point at %q", got, projectURL) + } +} diff --git a/parser.go b/parser.go index 6f8e014..77a6b3f 100644 --- a/parser.go +++ b/parser.go @@ -76,6 +76,15 @@ type ParserConfig struct { // Defaults to 30s if zero. Timeout time.Duration + // UserAgent is sent in the User-Agent header of every request made during + // the external checks. It defaults to [UserAgent], which names this library + // and its version. + // + // Programs embedding the parser are encouraged to identify themselves here, + // so that the administrators of the sites being checked can tell their + // traffic apart and allow it. + UserAgent string + // AllowNetworkToPrivateHosts allows the external checks to connect to // non-public addresses (loopback, private, link-local, ...). // @@ -120,9 +129,15 @@ func NewParser(config ParserConfig) (*Parser, error) { } // Hardened HTTP client: refuses connections to non-public addresses (SSRF) - // and caps the size of each response (resource exhaustion). See - // internal/safehttp.go. - httpClient := urlutil.SafeHTTPClient(timeout, config.AllowNetworkToPrivateHosts) + // and caps the size of each response (resource exhaustion). It also sets the + // User-Agent, which covers both the requests built here and the ones built + // by the internal httpclient and by go-vcsurl, since they all go through + // this client. See internal/safehttp.go. + httpClient := urlutil.SafeHTTPClient( + timeout, + config.AllowNetworkToPrivateHosts, + urlutil.WithUserAgent(config.UserAgent), + ) vcsurl.Client = httpClient p := Parser{ disableNetwork: config.DisableNetwork, diff --git a/publiccode-parser/publiccode_parser.go b/publiccode-parser/publiccode_parser.go index c1bb687..8a4d6e3 100644 --- a/publiccode-parser/publiccode_parser.go +++ b/publiccode-parser/publiccode_parser.go @@ -12,22 +12,29 @@ import ( publiccode "github.com/publiccodeyml/libpubliccode/v5" ) -var ( - version string - date string -) +var version, date = buildVersion() -func init() { - if version == "" { - version = "devel" - if info, ok := debug.ReadBuildInfo(); ok { - version = info.Main.Version - } +// buildVersion returns the module version and the commit date recorded in the +// build information of the running binary. +func buildVersion() (string, string) { + version, date := "devel", "(latest)" + + info, ok := debug.ReadBuildInfo() + if !ok { + return version, date } - if date == "" { - date = "(latest)" + if info.Main.Version != "" { + version = info.Main.Version } + + for _, setting := range info.Settings { + if setting.Key == "vcs.time" { + date = setting.Value + } + } + + return version, date } func main() { diff --git a/useragent.go b/useragent.go new file mode 100644 index 0000000..d848dca --- /dev/null +++ b/useragent.go @@ -0,0 +1,20 @@ +package publiccode + +import ( + urlutil "github.com/publiccodeyml/libpubliccode/v5/internal" +) + +// UserAgent returns the value the parser sends in the User-Agent header of the +// requests it makes during the external checks, e.g. +// +// libpubliccode/5.4.3 (+https://github.com/publiccodeyml/libpubliccode) +// +// The version comes from the build information of the running binary, and is +// "devel" when there is none to be found. +// +// Programs embedding the parser should identify themselves instead, through +// [ParserConfig.UserAgent]. This function is exported for the ones that want to +// keep this token and add their own to it. +func UserAgent() string { + return urlutil.UserAgent() +} diff --git a/useragent_test.go b/useragent_test.go new file mode 100644 index 0000000..1921104 --- /dev/null +++ b/useragent_test.go @@ -0,0 +1,96 @@ +package publiccode + +import ( + "net/http" + "net/http/httptest" + "net/url" + "slices" + "sync" + "testing" +) + +// userAgentRecorder is a server recording the User-Agent of every request it +// receives. +type userAgentRecorder struct { + *httptest.Server + + mu sync.Mutex + seen []string +} + +// newUserAgentRecorder starts a recording server, stopped when the test ends. +func newUserAgentRecorder(t *testing.T) *userAgentRecorder { + t.Helper() + + recorder := &userAgentRecorder{} + recorder.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + recorder.mu.Lock() + recorder.seen = append(recorder.seen, r.UserAgent()) + recorder.mu.Unlock() + + w.Header().Set("Content-Type", "text/yaml") + _, _ = w.Write([]byte("publiccodeYmlVersion: \"0.4\"\n")) + })) + + t.Cleanup(recorder.Close) + + return recorder +} + +// userAgents returns the User-Agents recorded so far. +func (r *userAgentRecorder) userAgents() []string { + r.mu.Lock() + defer r.mu.Unlock() + + return slices.Clone(r.seen) +} + +// TestUserAgentIsSentOnEveryRequestPath covers the two ways the parser reaches +// the network: the request it builds itself in Parse(), and the ones built by +// the internal httpclient during the external checks. +func TestUserAgentIsSentOnEveryRequestPath(t *testing.T) { + cases := map[string]string{ + "default": UserAgent(), + "override": "harvester/2.0 (+https://example.org)", + } + + for name, want := range cases { + t.Run(name, func(t *testing.T) { + srv := newUserAgentRecorder(t) + + config := ParserConfig{AllowNetworkToPrivateHosts: true} + if name == "override" { + config.UserAgent = want + } + + p, err := NewParser(config) + if err != nil { + t.Fatal(err) + } + + // Parse() builds its request itself. + _, _ = p.Parse(srv.URL + "/publiccode.yml") + + // The external checks go through the internal httpclient. + u, err := url.Parse(srv.URL + "/anything") + if err != nil { + t.Fatal(err) + } + + if _, err = p.isReachable(*u); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + seen := srv.userAgents() + if len(seen) < 2 { + t.Fatalf("expected both request paths to reach the server, got %d request(s)", len(seen)) + } + + for _, got := range seen { + if got != want { + t.Errorf("User-Agent received by the server: got %q, want %q", got, want) + } + } + }) + } +}