Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,12 @@ URL content. Package, audit, and Docker lookups use the configured
catalog from `data/ports.txt` and has a fixed bounded response; it has no
`max_length` setting.

Package metadata and OSV audit commands first use exact package/module names.
When an exact lookup fails, they perform a bounded best-effort search through
the relevant public package index and return suggestions; fuzzy candidates are
not automatically audited. Go module metadata uses the Go proxy's canonical
version field, including for `gopkg.in/...` module paths.

`daily` provides `!daily` in channels. Each authenticated account can claim
once per UTC calendar day, regardless of channel or network; users without an
account tag are limited by network and nickname. Different users can each
Expand Down
13 changes: 10 additions & 3 deletions docs/plugins.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,18 +112,23 @@ requests.
Use `!pkg go`, `!pkg npm`, or `!pkg pip` for current registry metadata, with an
optional version for a specific release. `!package` is an alias. The plugin
uses the public Go module proxy, npm registry, and PyPI endpoints, requires no
API keys, and bounds both request time and response length.
API keys, and bounds both request time and response length. The Go module proxy
response is handled using its canonical `Version` field, so paths such as
`gopkg.in/irc.v3` work correctly.

Examples:

~~~text
!pkg go github.com/variablenix/GoBot
!pkg npm lodash
!pkg pip requests 2.32.3
!pkg go irc
~~~

Responses include the registry version, a sanitized description when present,
and the canonical package page. `!package` is the only alias.
and the canonical package page. If an exact name is not found, GoBot performs a
bounded best-effort search and returns possible Go, npm, or PyPI matches with
links; use the suggested full name for metadata. `!package` is the only alias.

## Ports

Expand All @@ -144,7 +149,9 @@ key is required. With no version, the request omits the OSV `version` field,
then fetches the latest registry version and evaluates OSV affected ranges.
With a version, it performs an exact OSV query. Severity comes from OSV's
database-specific or severity fields, and fixed versions are shown when OSV
provides them.
provides them. If the package cannot be resolved exactly, GoBot returns a
bounded list of possible Go, npm, or PyPI package names; it does not audit all
fuzzy matches automatically.

## Docker Hub

Expand Down
3 changes: 2 additions & 1 deletion docs/security.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ dependencies, and deployment configuration maintained.
- The URL title plugin rejects loopback, private, link-local, multicast, and
local host targets to reduce SSRF risk.
- External HTTP lookups use timeouts and bound response sizes. Package, audit,
and Docker requests use fixed public provider hosts.
and Docker requests use fixed public provider hosts; package suggestions use
only the fixed public Go, npm, and PyPI index hosts.
- The paste plugin's URL mode is different: it fetches a user-supplied HTTP or
HTTPS URL from the bot host and follows redirects. Treat it as an outbound
network capability. Only enable it where users are trusted and host/network
Expand Down
29 changes: 28 additions & 1 deletion plugins/audit.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ type auditVulnerability struct {
func (p *Audit) Name() string { return "audit" }
func (p *Audit) Commands() []string { return []string{"audit", "vuln", "osv"} }
func (p *Audit) Help() string {
return "!audit <go|npm|pip> <package> [version] — discover known OSV vulnerabilities (aliases: !vuln, !osv)"
return "!audit <go|npm|pip> <package> [version] — discover known OSV vulnerabilities; failed exact names get fuzzy suggestions (aliases: !vuln, !osv)"
}
func (p *Audit) Init(c bot.PluginConfig, _ *storage.DB) error { p.cfg = c; return nil }

Expand Down Expand Up @@ -92,11 +92,21 @@ func (p *Audit) Handle(b *bot.Bot, m bot.Message) bool {
maxLength = configured
}
if version != "" {
if len(response.Vulns) == 0 {
if _, metadataErr := lookupPackageMetadata(ctx, ecosystem, parts[1], version); metadataErr == errPackageNotFound {
b.Send(m.ReplyTarget(), truncateRunes(formatAuditSuggestions(ctx, ecosystem, parts[1]), maxLength))
return true
}
}
b.Send(m.ReplyTarget(), truncateRunes(formatAuditExact(parts[1], version, response.Vulns, maxShown), maxLength))
return true
}
latest, err := lookupPackageMetadata(ctx, ecosystem, parts[1], "")
if err != nil {
if err == errPackageNotFound {
b.Send(m.ReplyTarget(), truncateRunes(formatAuditSuggestions(ctx, ecosystem, parts[1]), maxLength))
return true
}
b.Send(m.ReplyTarget(), "[audit] latest package version could not be determined")
return true
}
Expand All @@ -118,6 +128,23 @@ func (p *Audit) Handle(b *bot.Bot, m bot.Message) bool {
return true
}

func formatAuditSuggestions(ctx context.Context, ecosystem, query string) string {
query = cleanExternalText(query)
candidates, err := searchPackageCandidates(ctx, ecosystem, query)
if err != nil || len(candidates) == 0 {
return fmt.Sprintf("[audit] %s not found; use the full package/module name", query)
}
items := make([]string, 0, len(candidates))
for _, candidate := range candidates {
name := cleanExternalText(candidate.Name)
if candidate.Version != "" {
name += " " + cleanExternalText(candidate.Version)
}
items = append(items, name)
}
return fmt.Sprintf("[audit] no exact match for %s; possible packages: %s", query, strings.Join(items, "; "))
}

func queryOSV(ctx context.Context, ecosystem, name, version string) (osvResponse, error) {
request := map[string]interface{}{"package": map[string]string{"name": name, "ecosystem": ecosystem}}
if version != "" {
Expand Down
12 changes: 12 additions & 0 deletions plugins/audit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,15 @@ func TestAuditFormatsCVESeverityFixedAndMore(t *testing.T) {
t.Fatal("OSV range matching is incorrect")
}
}

func TestFormatAuditSuggestionsUsesSearchResults(t *testing.T) {
old := apiHTTPClient
t.Cleanup(func() { apiHTTPClient = old })
apiHTTPClient = &http.Client{Transport: newPluginRoundTripper(func(r *http.Request) (*http.Response, error) {
return newPluginResponse(http.StatusOK, `{"objects":[{"package":{"name":"libirc-client","version":"0.0.3"}}]}`), nil
})}
got := formatAuditSuggestions(t.Context(), "npm", "libirc")
if !strings.Contains(got, "possible packages: libirc-client 0.0.3") {
t.Fatalf("audit suggestion output = %q", got)
}
}
201 changes: 198 additions & 3 deletions plugins/pkg.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/variablenix/GoBot/bot"
"github.com/variablenix/GoBot/storage"
"golang.org/x/net/html"
)

type Pkg struct{ cfg bot.PluginConfig }
Expand All @@ -23,10 +24,17 @@ type packageMetadata struct {
Description string
}

type packageCandidate struct {
Name string
Version string
Description string
URL string
}

func (p *Pkg) Name() string { return "pkg" }
func (p *Pkg) Commands() []string { return []string{"pkg", "package"} }
func (p *Pkg) Help() string {
return "!pkg <go|npm|pip> <package> [version] — show package metadata (alias: !package)"
return "!pkg <go|npm|pip> <package> [version] — show package metadata; failed exact names get fuzzy suggestions (alias: !package)"
}
func (p *Pkg) Init(c bot.PluginConfig, _ *storage.DB) error { p.cfg = c; return nil }

Expand Down Expand Up @@ -55,7 +63,7 @@ func (p *Pkg) Handle(b *bot.Bot, m bot.Message) bool {
metadata, err := lookupPackageMetadata(ctx, ecosystem, parts[1], version)
if err != nil {
if err == errPackageNotFound {
b.Send(m.ReplyTarget(), fmt.Sprintf("[%s] %s not found", ecosystem, cleanExternalText(parts[1])))
b.Send(m.ReplyTarget(), truncateRunes(formatPackageSuggestions(ctx, ecosystem, parts[1]), packageMaxLength(p.cfg)))
} else {
b.Send(m.ReplyTarget(), fmt.Sprintf("[%s] package lookup is temporarily unavailable", ecosystem))
}
Expand Down Expand Up @@ -141,7 +149,7 @@ func lookupPackageMetadata(ctx context.Context, ecosystem, name, version string)
metadata.Version, _ = info["version"].(string)
metadata.Description, _ = info["summary"].(string)
} else {
metadata.Version, _ = payload["version"].(string)
metadata.Version = packagePayloadString(payload, "version", "Version")
metadata.Description, _ = payload["description"].(string)
}
if metadata.Version == "" {
Expand All @@ -150,6 +158,193 @@ func lookupPackageMetadata(ctx context.Context, ecosystem, name, version string)
return metadata, nil
}

func packagePayloadString(payload map[string]interface{}, keys ...string) string {
for _, key := range keys {
if value, ok := payload[key].(string); ok && strings.TrimSpace(value) != "" {
return value
}
}
return ""
}

func formatPackageSuggestions(ctx context.Context, ecosystem, query string) string {
query = cleanExternalText(query)
candidates, err := searchPackageCandidates(ctx, ecosystem, query)
if err != nil || len(candidates) == 0 {
return fmt.Sprintf("[%s] %s not found; use the full package/module name", ecosystem, query)
}
items := make([]string, 0, len(candidates))
for _, candidate := range candidates {
name := cleanExternalText(candidate.Name)
if candidate.Version != "" {
name += " " + cleanExternalText(candidate.Version)
}
if candidate.URL != "" {
name += " (" + cleanExternalText(candidate.URL) + ")"
}
items = append(items, name)
}
return fmt.Sprintf("[%s] no exact match for %s; possible matches: %s", ecosystem, query, strings.Join(items, "; "))
}

func searchPackageCandidates(ctx context.Context, ecosystem, query string) ([]packageCandidate, error) {
if query == "" || len([]rune(query)) > 240 {
return nil, errPackageNotFound
}
switch ecosystem {
case "Go":
return searchGoPackages(ctx, query)
case "npm":
return searchNPMPackages(ctx, query)
case "PyPI":
return searchPyPIPackages(ctx, query)
default:
return nil, errPackageNotFound
}
}

func searchGoPackages(ctx context.Context, query string) ([]packageCandidate, error) {
endpoint := "https://pkg.go.dev/search?m=package&limit=5&q=" + url.QueryEscape(query)
body, err := getPackageResponse(ctx, endpoint, 2*1024*1024)
if err != nil {
return nil, err
}
doc, err := html.Parse(strings.NewReader(string(body)))
if err != nil {
return nil, err
}
candidates := make([]packageCandidate, 0, 5)
walkHTML(doc, func(node *html.Node) {
if len(candidates) >= 5 || node.Type != html.ElementNode || node.Data != "a" || !hasHTMLAttribute(node, "data-test-id", "snippet-title") {
return
}
href := htmlAttribute(node, "href")
if !strings.HasPrefix(href, "/") || strings.ContainsAny(href, "?#\r\n") {
return
}
module := strings.TrimPrefix(href, "/")
if module == "" {
return
}
candidates = append(candidates, packageCandidate{Name: module, URL: "https://pkg.go.dev/" + module})
})
return candidates, nil
}

func searchNPMPackages(ctx context.Context, query string) ([]packageCandidate, error) {
endpoint := "https://registry.npmjs.org/-/v1/search?text=" + url.QueryEscape(query) + "&size=5"
body, err := getPackageResponse(ctx, endpoint, 2*1024*1024)
if err != nil {
return nil, err
}
var response struct {
Objects []struct {
Package struct {
Name string `json:"name"`
Version string `json:"version"`
Description string `json:"description"`
Links struct {
NPM string `json:"npm"`
} `json:"links"`
} `json:"package"`
} `json:"objects"`
}
if err := json.Unmarshal(body, &response); err != nil {
return nil, err
}
candidates := make([]packageCandidate, 0, len(response.Objects))
for _, object := range response.Objects {
if object.Package.Name == "" {
continue
}
link := object.Package.Links.NPM
if link == "" {
link = "https://www.npmjs.com/package/" + url.PathEscape(object.Package.Name)
}
candidates = append(candidates, packageCandidate{Name: object.Package.Name, Version: object.Package.Version, Description: object.Package.Description, URL: link})
}
return candidates, nil
}

func searchPyPIPackages(ctx context.Context, query string) ([]packageCandidate, error) {
endpoint := "https://pypi.org/search/?q=" + url.QueryEscape(query)
body, err := getPackageResponse(ctx, endpoint, 2*1024*1024)
if err != nil {
return nil, err
}
doc, err := html.Parse(strings.NewReader(string(body)))
if err != nil {
return nil, err
}
candidates := make([]packageCandidate, 0, 5)
walkHTML(doc, func(node *html.Node) {
if len(candidates) >= 5 || node.Type != html.ElementNode || node.Data != "a" || !hasHTMLClass(node, "package-snippet") {
return
}
href := htmlAttribute(node, "href")
prefix := "/project/"
if !strings.HasPrefix(href, prefix) {
return
}
name := strings.Trim(strings.TrimPrefix(href, prefix), "/")
if name == "" || strings.ContainsAny(name, "?#\r\n") {
return
}
candidates = append(candidates, packageCandidate{Name: name, URL: "https://pypi.org/project/" + name + "/"})
})
return candidates, nil
}

func getPackageResponse(ctx context.Context, endpoint string, limit int64) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json, text/html")
req.Header.Set("User-Agent", "GoBot/1.0 (IRC bot; package search)")
res, err := apiHTTPClient.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, fmt.Errorf("package search returned HTTP %d", res.StatusCode)
}
return io.ReadAll(io.LimitReader(res.Body, limit))
}

func walkHTML(node *html.Node, visit func(*html.Node)) {
if node == nil {
return
}
visit(node)
for child := node.FirstChild; child != nil; child = child.NextSibling {
walkHTML(child, visit)
}
}

func htmlAttribute(node *html.Node, key string) string {
for _, attribute := range node.Attr {
if attribute.Key == key {
return strings.TrimSpace(attribute.Val)
}
}
return ""
}

func hasHTMLAttribute(node *html.Node, key, value string) bool {
return htmlAttribute(node, key) == value
}

func hasHTMLClass(node *html.Node, class string) bool {
for _, value := range strings.Fields(htmlAttribute(node, "class")) {
if value == class {
return true
}
}
return false
}

func packagePath(value string) string {
parts := strings.Split(strings.Trim(value, "/"), "/")
for i := range parts {
Expand Down
Loading
Loading