Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 14 additions & 23 deletions internal/database/queries.go
Original file line number Diff line number Diff line change
Expand Up @@ -443,11 +443,14 @@ func (db *DB) GetMostPopularPackages(limit int) ([]PopularPackage, error) {
}

type RecentPackage struct {
Ecosystem string `db:"ecosystem"`
Name string `db:"name"`
Version string `db:"version"`
CachedAt time.Time `db:"fetched_at"`
Size int64 `db:"size"`
Ecosystem string `db:"ecosystem"`
Name string `db:"name"`
VersionPURL string `db:"version_purl"`
CachedAt time.Time `db:"fetched_at"`
Size int64 `db:"size"`
// Version is derived from VersionPURL rather than selected, so that the
// PURL percent-encoding is decoded (e.g. "%2B" back to "+").
Version string `db:"-"`
}

func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) {
Expand All @@ -461,10 +464,10 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) {
}

var packages []RecentPackage
// We need to extract version from the purl since there's no separate version column
// There is no separate version column, so the full version PURL is selected
// and the version is decoded from it in Go.
query := db.Rebind(`
SELECT p.ecosystem, p.name,
SUBSTR(v.purl, INSTR(v.purl, '@') + 1) as version,
SELECT p.ecosystem, p.name, v.purl as version_purl,
a.fetched_at, COALESCE(a.size, 0) as size
FROM artifacts a
JOIN versions v ON v.purl = a.version_purl
Expand All @@ -474,25 +477,13 @@ func (db *DB) GetRecentlyCachedPackages(limit int) ([]RecentPackage, error) {
LIMIT ?
`)

// For postgres, use different string function
if db.dialect == DialectPostgres {
query = db.Rebind(`
SELECT p.ecosystem, p.name,
SUBSTRING(v.purl FROM POSITION('@' IN v.purl) + 1) as version,
a.fetched_at, COALESCE(a.size, 0) as size
FROM artifacts a
JOIN versions v ON v.purl = a.version_purl
JOIN packages p ON p.purl = v.package_purl
WHERE a.storage_path IS NOT NULL AND a.fetched_at IS NOT NULL
ORDER BY a.fetched_at DESC
LIMIT ?
`)
}

err = db.Select(&packages, query, limit)
if err != nil {
return nil, err
}
for i := range packages {
packages[i].Version = VersionFromPURL(packages[i].VersionPURL)
}
return packages, nil
}

Expand Down
76 changes: 73 additions & 3 deletions internal/database/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package database

import (
"database/sql"
"net/url"
"strings"
"time"
)
Expand Down Expand Up @@ -47,10 +48,79 @@ type Version struct {
// Version extracts the version string from the PURL.
// e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21"
func (v *Version) Version() string {
if idx := strings.LastIndex(v.PURL, "@"); idx >= 0 {
return v.PURL[idx+1:]
return VersionFromPURL(v.PURL)
}

// EscapedVersion returns the version escaped for use as a single URL path
// segment.
//
// Version returns decoded text, which is what should be shown to a user but is
// not safe to drop into a link: html/template preserves reserved characters and
// existing escapes in a URL, so "release/1" would split into two path segments,
// "v1?build" would start a query string, and a literal "%2B" would be read back
// as "+". Escaping here and decoding in splitWildcardPath round-trips the value,
// so the link resolves to the version that was stored.
func (v *Version) EscapedVersion() string {
return url.PathEscape(v.Version())
}

// DisplayPURL returns the PURL with its path components percent-decoded, for
// showing in the UI. The stored PURL keeps the canonical encoding (which is
// what the API and all lookups use); this is only a readable rendering, so that
// a version like "7.91+dfsg1-2ubuntu0.1" is not shown as "7.91%2Bdfsg1-2ubuntu0.1"
// and an npm scope is shown as "@babel" rather than "%40babel". Qualifiers and
// subpath keep their encoding, since decoding those would be ambiguous.
func (v *Version) DisplayPURL() string {
base, suffix := v.PURL, ""
if i := strings.IndexAny(base, "?#"); i >= 0 {
base, suffix = base[:i], base[i:]
}

name, version := base, ""
if idx := strings.LastIndex(base, "@"); idx >= 0 {
name, version = base[:idx], "@"+decodePURLComponent(base[idx+1:])
}

parts := strings.Split(name, "/")
for i, part := range parts {
parts[i] = decodePURLComponent(part)
}
return strings.Join(parts, "/") + version + suffix
}

// VersionFromPURL extracts the decoded version string from a PURL.
//
// PURL percent-encodes characters that are not safe in a path component, so a
// Debian version like "7.91+dfsg1-2ubuntu0.1" is stored as
// "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1". The raw substring after "@" is
// therefore not the version: it must be percent-decoded before being displayed
// or used to build a URL, otherwise "%2B" leaks into the UI and round-tripping
// the value back into a PURL double-encodes it.
//
// e.g., "pkg:npm/lodash@4.17.21" -> "4.17.21"
func VersionFromPURL(p string) string {
// Qualifiers ("?key=value") and subpath ("#path") follow the version.
if i := strings.IndexAny(p, "?#"); i >= 0 {
p = p[:i]
}
idx := strings.LastIndex(p, "@")
if idx < 0 {
return ""
}
return decodePURLComponent(p[idx+1:])
}

// decodePURLComponent percent-decodes a single PURL path component, returning
// the input unchanged if it is not valid percent-encoding.
func decodePURLComponent(s string) string {
if !strings.Contains(s, "%") {
return s
}
decoded, err := url.PathUnescape(s)
if err != nil {
return s
}
return ""
return decoded
}

// Artifact represents a cached artifact in the database.
Expand Down
159 changes: 159 additions & 0 deletions internal/database/version_purl_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
package database

import (
"database/sql"
"net/url"
"testing"
"time"
)

func TestVersionFromPURL(t *testing.T) {
tests := []struct {
name string
purl string
want string
}{
{"simple", "pkg:npm/lodash@4.17.21", "4.17.21"},
{"namespaced", "pkg:composer/symfony/console@6.0.0", "6.0.0"},
// Debian/Ubuntu versions routinely contain "+", which PURL encodes.
{"encoded plus", "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"},
{"encoded epoch", "pkg:deb/curl@1%3A7.81.0-1", "1:7.81.0-1"},
{"encoded plus with qualifier", "pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com", "7.91+dfsg1"},
{"tilde is not encoded", "pkg:deb/foo@1.0~rc1", "1.0~rc1"},
{"no version", "pkg:npm/lodash", ""},
{"invalid escape passed through", "pkg:npm/lodash@1.0%zz", "1.0%zz"},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := VersionFromPURL(tt.purl); got != tt.want {
t.Errorf("VersionFromPURL(%q) = %q, want %q", tt.purl, got, tt.want)
}
v := &Version{PURL: tt.purl}
if got := v.Version(); got != tt.want {
t.Errorf("Version.Version() for %q = %q, want %q", tt.purl, got, tt.want)
}
})
}
}

// TestVersionEscapedVersion checks the value the templates put in a URL. It
// must survive the round trip back through the router: escaping here and
// decoding per path segment on the way in has to yield the original version.
func TestVersionEscapedVersion(t *testing.T) {
tests := []struct {
name string
purl string
want string
}{
{"simple", "pkg:npm/lodash@4.17.21", "4.17.21"},
// "+" is legal in a path segment, so it stays literal and the UI keeps
// showing the version the way Debian writes it.
{"plus stays literal", "pkg:deb/nmap@7.91%2Bdfsg1-2ubuntu0.1", "7.91+dfsg1-2ubuntu0.1"},
// A slash would otherwise split the version into two path segments.
{"slash", "pkg:golang/example@release%2F1", "release%2F1"},
// A question mark would otherwise start the query string.
{"question mark", "pkg:npm/example@v1%3Fbuild", "v1%3Fbuild"},
// A version containing a literal "%2B" is stored double-encoded; the
// link must re-encode it or it decodes back to "+" instead.
{"literal percent escape", "pkg:npm/example@1.0%252B", "1.0%252B"},
{"space", "pkg:npm/example@1.0%20beta", "1.0%20beta"},
{"no version", "pkg:npm/lodash", ""},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
v := &Version{PURL: tt.purl}
got := v.EscapedVersion()
if got != tt.want {
t.Errorf("EscapedVersion() for %q = %q, want %q", tt.purl, got, tt.want)
}
// The router decodes each path segment, which must give back the
// version the page displays.
decoded, err := url.PathUnescape(got)
if err != nil {
t.Fatalf("PathUnescape(%q) failed: %v", got, err)
}
if decoded != v.Version() {
t.Errorf("round trip for %q = %q, want %q", tt.purl, decoded, v.Version())
}
})
}
}

func TestVersionDisplayPURL(t *testing.T) {
tests := []struct {
name string
purl string
want string
}{
{"simple", "pkg:npm/lodash@4.17.21", "pkg:npm/lodash@4.17.21"},
{
"encoded plus",
"pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1",
"pkg:deb/nmap@7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1",
},
{
"qualifier preserved",
"pkg:deb/nmap@7.91%2Bdfsg1?repository_url=http%3A%2F%2Fexample.com",
"pkg:deb/nmap@7.91+dfsg1?repository_url=http%3A%2F%2Fexample.com",
},
// The namespace is encoded too: MakePURLString("npm", "@babel/core", …)
// produces "pkg:npm/%40babel/core@…".
{"encoded npm scope", "pkg:npm/%40babel/core@7.0.0", "pkg:npm/@babel/core@7.0.0"},
{"encoded scope without version", "pkg:npm/%40babel/core", "pkg:npm/@babel/core"},
{"no version", "pkg:npm/lodash", "pkg:npm/lodash"},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
v := &Version{PURL: tt.purl}
if got := v.DisplayPURL(); got != tt.want {
t.Errorf("DisplayPURL() for %q = %q, want %q", tt.purl, got, tt.want)
}
})
}
}

// TestGetRecentlyCachedPackagesDecodesVersion guards the dashboard's "recently
// cached" list, which derives the version from the version PURL.
func TestGetRecentlyCachedPackagesDecodesVersion(t *testing.T) {
runWithBothDatabases(t, func(t *testing.T, db *DB) {
const versionPURL = "pkg:deb/nmap@7.91%2Bdfsg1%2Breally7.80%2Bdfsg1-2ubuntu0.1"

if err := db.UpsertPackage(&Package{
PURL: "pkg:deb/nmap", Ecosystem: "deb", Name: "nmap",
}); err != nil {
t.Fatalf("UpsertPackage failed: %v", err)
}
if err := db.UpsertVersion(&Version{
PURL: versionPURL, PackagePURL: "pkg:deb/nmap",
}); err != nil {
t.Fatalf("UpsertVersion failed: %v", err)
}
if err := db.UpsertArtifact(&Artifact{
VersionPURL: versionPURL,
Filename: "nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb",
UpstreamURL: "http://archive.ubuntu.com/ubuntu/pool/universe/n/nmap/nmap.deb",
StoragePath: sql.NullString{String: "/cache/nmap.deb", Valid: true},
FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
}); err != nil {
t.Fatalf("UpsertArtifact failed: %v", err)
}

recent, err := db.GetRecentlyCachedPackages(10)
if err != nil {
t.Fatalf("GetRecentlyCachedPackages failed: %v", err)
}
if len(recent) != 1 {
t.Fatalf("expected 1 recent package, got %d", len(recent))
}
const want = "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1"
if recent[0].Version != want {
t.Errorf("Version = %q, want %q", recent[0].Version, want)
}
if recent[0].VersionPURL != versionPURL {
t.Errorf("VersionPURL = %q, want %q", recent[0].VersionPURL, versionPURL)
}
})
}
5 changes: 5 additions & 0 deletions internal/handler/debian_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ func TestDebianHandler_parsePoolPath(t *testing.T) {
{"pool/main/libn/libncurses/libncurses6_6.2-1_amd64.deb", "libncurses6", "6.2-1", "amd64"},
{"pool/contrib/v/virtualbox/virtualbox_6.1.38-1_amd64.deb", "virtualbox", "6.1.38-1", "amd64"},
{"pool/main/g/git/git_2.39.2-1_arm64.deb", "git", "2.39.2-1", "arm64"},
{
"pool/universe/n/nmap/nmap_7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1_amd64.deb",
"nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", "amd64",
},
{"pool/main/o/openssl/openssl_3.0.2-0ubuntu1.15~build1_amd64.deb", "openssl", "3.0.2-0ubuntu1.15~build1", "amd64"},
{"invalid/path", "", "", ""},
{"pool/main/n/nginx/nginx.deb", "", "", ""},
})
Expand Down
10 changes: 4 additions & 6 deletions internal/server/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,12 +139,11 @@ type BulkResponse struct {
// Resolves namespaced package names (Composer vendor/name, npm @scope/name) from the path.
func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem")
wildcard := chi.URLParam(r, "*")
if err := validatePackagePath(wildcard); err != nil {
segments, err := packagePathSegments(r)
if err != nil {
badRequest(w, err.Error())
return
}
segments := splitWildcardPath(wildcard)

if ecosystem == "" || len(segments) == 0 {
badRequest(w, "ecosystem and name are required")
Expand Down Expand Up @@ -277,12 +276,11 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste
// Supports both {name} and {name}/{version} paths with namespaced package names.
func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem")
wildcard := chi.URLParam(r, "*")
if err := validatePackagePath(wildcard); err != nil {
segments, err := packagePathSegments(r)
if err != nil {
badRequest(w, err.Error())
return
}
segments := splitWildcardPath(wildcard)

if ecosystem == "" || len(segments) == 0 {
badRequest(w, "ecosystem and name are required")
Expand Down
Loading