diff --git a/Gopkg.lock b/Gopkg.lock index 1687513c..7f562c18 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -733,10 +733,11 @@ [[projects]] branch = "master" - digest = "1:35d68717797810014f26d6a175b0e73bd7fba070c408d443b3fff87a61c8008e" + digest = "1:25c965216c188afcd5f430f65acc28d4f193c5a88e7c6c54ae876b1898f86368" name = "golang.org/x/net" packages = [ "context", + "context/ctxhttp", "http/httpguts", "http2", "http2/hpack", @@ -748,6 +749,17 @@ pruneopts = "UT" revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc" +[[projects]] + branch = "master" + digest = "1:e007b54f54cbd4214aa6d97a67d57bc2539991adb4e22ea92c482bbece8de469" + name = "golang.org/x/oauth2" + packages = [ + ".", + "internal", + ] + pruneopts = "UT" + revision = "99b60b757ec124ebb7d6b7e97f153b19c10ce163" + [[projects]] branch = "master" digest = "1:75515eedc0dc2cb0b40372008b616fa2841d831c63eedd403285ff286c593295" @@ -799,9 +811,18 @@ version = "v0.1.0" [[projects]] - digest = "1:c25289f43ac4a68d88b02245742347c94f1e108c534dda442188015ff80669b3" + digest = "1:9e29a0ec029d012437d88da3ccccf18adcdce069cab08d462056c2c6bb006505" name = "google.golang.org/appengine" - packages = ["cloudsql"] + packages = [ + "cloudsql", + "internal", + "internal/base", + "internal/datastore", + "internal/log", + "internal/remote_api", + "internal/urlfetch", + "urlfetch", + ] pruneopts = "UT" revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" version = "v1.4.0" @@ -964,6 +985,7 @@ "github.com/sirupsen/logrus", "golang.org/x/crypto/ssh", "golang.org/x/crypto/ssh/agent", + "golang.org/x/oauth2", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/commands/report.go b/commands/report.go index aa862bc7..1b3691c5 100644 --- a/commands/report.go +++ b/commands/report.go @@ -64,6 +64,7 @@ func (*ReportCmd) Usage() string { [-diff] [-ignore-unscored-cves] [-ignore-unfixed] + [-ignore-github-dismissed] [-to-email] [-to-http] [-to-slack] @@ -133,10 +134,12 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&c.Conf.IgnoreUnscoredCves, "ignore-unscored-cves", false, "Don't report the unscored CVEs") - f.BoolVar( - &c.Conf.IgnoreUnfixed, "ignore-unfixed", false, + f.BoolVar(&c.Conf.IgnoreUnfixed, "ignore-unfixed", false, "Don't report the unfixed CVEs") + f.BoolVar(&c.Conf.IgnoreGitHubDismissed, "ignore-github-dismissed", false, + "Don't report the dismissed CVEs on GitHub Security Alerts") + f.StringVar( &c.Conf.HTTPProxy, "http-proxy", "", "http://proxy-url:port (default: empty)") diff --git a/config/config.go b/config/config.go index cd8e5266..237df6ff 100644 --- a/config/config.go +++ b/config/config.go @@ -98,31 +98,34 @@ const ( //Config is struct of Configuration type Config struct { - Debug bool `json:"debug"` - DebugSQL bool `json:"debugSQL"` - Lang string `json:"lang"` - HTTPProxy string `valid:"url" json:"httpProxy"` - LogDir string `json:"logDir"` - ResultsDir string `json:"resultsDir"` - Pipe bool `json:"pipe"` + Debug bool `json:"debug,omitempty"` + DebugSQL bool `json:"debugSQL,omitempty"` + Lang string `json:"lang,omitempty"` + HTTPProxy string `valid:"url" json:"httpProxy,omitempty"` + LogDir string `json:"logDir,omitempty"` + ResultsDir string `json:"resultsDir,omitempty"` + Pipe bool `json:"pipe,omitempty"` - Default ServerInfo `json:"default"` - Servers map[string]ServerInfo `json:"servers"` - CvssScoreOver float64 `json:"cvssScoreOver"` - IgnoreUnscoredCves bool `json:"ignoreUnscoredCves"` - IgnoreUnfixed bool `json:"ignoreUnfixed"` - SSHNative bool `json:"sshNative"` - SSHConfig bool `json:"sshConfig"` - ContainersOnly bool `json:"containersOnly"` - SkipBroken bool `json:"skipBroken"` - CacheDBPath string `json:"cacheDBPath"` - Vvv bool `json:"vvv"` - UUID bool `json:"uuid"` + Default ServerInfo `json:"default,omitempty"` + Servers map[string]ServerInfo `json:"servers,omitempty"` + CvssScoreOver float64 `json:"cvssScoreOver,omitempty"` - CveDict GoCveDictConf `json:"cveDict"` - OvalDict GovalDictConf `json:"ovalDict"` - Gost GostConf `json:"gost"` - Exploit ExploitConf `json:"exploit"` + IgnoreUnscoredCves bool `json:"ignoreUnscoredCves,omitempty"` + IgnoreUnfixed bool `json:"ignoreUnfixed,omitempty"` + IgnoreGitHubDismissed bool `json:"ignore_git_hub_dismissed,omitempty"` + + SSHNative bool `json:"sshNative,omitempty"` + SSHConfig bool `json:"sshConfig,omitempty"` + ContainersOnly bool `json:"containersOnly,omitempty"` + SkipBroken bool `json:"skipBroken,omitempty"` + CacheDBPath string `json:"cacheDBPath,omitempty"` + Vvv bool `json:"vvv,omitempty"` + UUID bool `json:"uuid,omitempty"` + + CveDict GoCveDictConf `json:"cveDict,omitempty"` + OvalDict GovalDictConf `json:"ovalDict,omitempty"` + Gost GostConf `json:"gost,omitempty"` + Exploit ExploitConf `json:"exploit,omitempty"` Slack SlackConf `json:"-"` EMail SMTPConf `json:"-"` @@ -136,27 +139,27 @@ type Config struct { Telegram TelegramConf `json:"-"` Saas SaasConf `json:"-"` - RefreshCve bool `json:"refreshCve"` - ToSlack bool `json:"toSlack"` - ToStride bool `json:"toStride"` - ToHipChat bool `json:"toHipChat"` - ToChatWork bool `json:"toChatWork"` - ToTelegram bool `json:"ToTelegram"` - ToEmail bool `json:"toEmail"` - ToSyslog bool `json:"toSyslog"` - ToLocalFile bool `json:"toLocalFile"` - ToS3 bool `json:"toS3"` - ToAzureBlob bool `json:"toAzureBlob"` - ToSaas bool `json:"toSaas"` - ToHTTP bool `json:"toHTTP"` - FormatXML bool `json:"formatXML"` - FormatJSON bool `json:"formatJSON"` - FormatOneEMail bool `json:"formatOneEMail"` - FormatOneLineText bool `json:"formatOneLineText"` - FormatList bool `json:"formatList"` - FormatFullText bool `json:"formatFullText"` - GZIP bool `json:"gzip"` - Diff bool `json:"diff"` + RefreshCve bool `json:"refreshCve,omitempty"` + ToSlack bool `json:"toSlack,omitempty"` + ToStride bool `json:"toStride,omitempty"` + ToHipChat bool `json:"toHipChat,omitempty"` + ToChatWork bool `json:"toChatWork,omitempty"` + ToTelegram bool `json:"ToTelegram,omitempty"` + ToEmail bool `json:"toEmail,omitempty"` + ToSyslog bool `json:"toSyslog,omitempty"` + ToLocalFile bool `json:"toLocalFile,omitempty"` + ToS3 bool `json:"toS3,omitempty"` + ToAzureBlob bool `json:"toAzureBlob,omitempty"` + ToSaas bool `json:"toSaas,omitempty"` + ToHTTP bool `json:"toHTTP,omitempty"` + FormatXML bool `json:"formatXML,omitempty"` + FormatJSON bool `json:"formatJSON,omitempty"` + FormatOneEMail bool `json:"formatOneEMail,omitempty"` + FormatOneLineText bool `json:"formatOneLineText,omitempty"` + FormatList bool `json:"formatList,omitempty"` + FormatFullText bool `json:"formatFullText,omitempty"` + GZIP bool `json:"gzip,omitempty"` + Diff bool `json:"diff,omitempty"` } // ValidateOnConfigtest validates @@ -1054,6 +1057,7 @@ type ServerInfo struct { Containers map[string]ContainerSetting `toml:"containers" json:"containers,omitempty"` IgnoreCves []string `toml:"ignoreCves,omitempty" json:"ignoreCves,omitempty"` IgnorePkgsRegexp []string `toml:"ignorePkgsRegexp,omitempty" json:"ignorePkgsRegexp,omitempty"` + GitHubRepos map[string]GitHubConf `toml:"githubs" json:"githubs,omitempty"` // key: owner/repo UUIDs map[string]string `toml:"uuids,omitempty" json:"uuids,omitempty"` Memo string `toml:"memo,omitempty" json:"memo"` Enablerepo []string `toml:"enablerepo,omitempty" json:"enablerepo,omitempty"` // For CentOS, RHEL, Amazon @@ -1077,6 +1081,23 @@ type ContainerSetting struct { IgnoreCves []string `json:"ignoreCves,omitempty"` } +// IntegrationConf is used for integration configuration +type IntegrationConf struct { + GitHubConf map[string]GitHubConf +} + +// New creates IntegrationConf and initialize fields +func (c IntegrationConf) New() IntegrationConf { + return IntegrationConf{ + GitHubConf: map[string]GitHubConf{}, + } +} + +// GitHubConf is used for GitHub integration +type GitHubConf struct { + Token string `json:"token"` +} + // ScanMode has a type of scan mode. fast, fast-root, deep and offline type ScanMode struct { flag byte diff --git a/config/tomlloader.go b/config/tomlloader.go index 6b657e79..937fd675 100644 --- a/config/tomlloader.go +++ b/config/tomlloader.go @@ -254,6 +254,18 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error { } } + s.GitHubRepos = v.GitHubRepos + for ownerRepo, githubSetting := range s.GitHubRepos { + if ss := strings.Split(ownerRepo, "/"); len(ss) != 2 { + return fmt.Errorf("Failed to parse GitHub owner/repo: %s in %s", + ownerRepo, serverName) + } + if githubSetting.Token == "" { + return fmt.Errorf("GitHub owner/repo: %s in %s token is empty", + ownerRepo, serverName) + } + } + s.UUIDs = v.UUIDs s.Type = v.Type diff --git a/github/github.go b/github/github.go new file mode 100644 index 00000000..a4c1ff2b --- /dev/null +++ b/github/github.go @@ -0,0 +1,144 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Corporation , Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package github + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + "github.com/k0kubun/pp" + "golang.org/x/oauth2" +) + +// FillGitHubSecurityAlerts access to owner/repo on GitHub and fetch scurity alerts of the repository via GitHub API v4 GraphQL and then set to the given ScanResult. +// https://help.github.com/articles/about-security-alerts-for-vulnerable-dependencies/ +func FillGitHubSecurityAlerts(r *models.ScanResult, owner, repo, token string) (nCVEs int, err error) { + src := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + httpClient := oauth2.NewClient(context.Background(), src) + + // TODO Use `https://github.com/shurcooL/githubv4` if the tool supports vulnerabilityAlerts Endpoint + const jsonfmt = `{"query": + "query { repository(owner:\"%s\", name:\"%s\") { url, vulnerabilityAlerts(first: %d, %s) { pageInfo{ endCursor, hasNextPage, startCursor}, edges { node { id, externalIdentifier, externalReference, fixedIn, packageName, dismissReason, dismissedAt } } } } }"}` + after := "" + + for { + jsonStr := fmt.Sprintf(jsonfmt, owner, repo, 100, after) + req, err := http.NewRequest("POST", + "https://api.github.com/graphql", + bytes.NewBuffer([]byte(jsonStr)), + ) + if err != nil { + return 0, err + } + + // https://developer.github.com/v4/previews/#repository-vulnerability-alerts + // To toggle this preview and access data, need to provide a custom media type in the Accept header: + // MEMO: I tried to get the affected version via GitHub API. Bit it seems difficult to determin the affected version if there are multiple dependency files such as package.json. + // TODO remove this header if it is no longer preview status in the future. + req.Header.Set("Accept", "application/vnd.github.vixen-preview+json") + req.Header.Set("Content-Type", "application/json") + + resp, err := httpClient.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + alerts := SecurityAlerts{} + if json.NewDecoder(resp.Body).Decode(&alerts); err != nil { + return 0, err + } + + util.Log.Debugf("%s", pp.Sprint(alerts)) + + for _, v := range alerts.Data.Repository.VulnerabilityAlerts.Edges { + if config.Conf.IgnoreGitHubDismissed && v.Node.DismissReason != "" { + continue + } + + pkgName := fmt.Sprintf("%s %s", + alerts.Data.Repository.URL, v.Node.PackageName) + + m := models.GitHubSecurityAlert{ + PackageName: pkgName, + FixedIn: v.Node.FixedIn, + AffectedRange: v.Node.AffectedRange, + Dismissed: len(v.Node.DismissReason) != 0, + DismissedAt: v.Node.DismissedAt, + DismissReason: v.Node.DismissReason, + } + + cveID := v.Node.ExternalIdentifier + + if val, ok := r.ScannedCves[cveID]; ok { + val.GitHubSecurityAlerts = val.GitHubSecurityAlerts.Add(m) + r.ScannedCves[cveID] = val + nCVEs++ + } else { + v := models.VulnInfo{ + CveID: cveID, + Confidences: models.Confidences{models.GitHubMatch}, + GitHubSecurityAlerts: models.GitHubSecurityAlerts{m}, + } + r.ScannedCves[cveID] = v + nCVEs++ + } + } + if !alerts.Data.Repository.VulnerabilityAlerts.PageInfo.HasNextPage { + break + } + after = fmt.Sprintf(`after: \"%s\"`, alerts.Data.Repository.VulnerabilityAlerts.PageInfo.EndCursor) + } + return nCVEs, err +} + +//SecurityAlerts has detected CVE-IDs, PackageNames, Refs +type SecurityAlerts struct { + Data struct { + Repository struct { + URL string `json:"url,omitempty"` + VulnerabilityAlerts struct { + PageInfo struct { + EndCursor string `json:"endCursor,omitempty"` + HasNextPage bool `json:"hasNextPage,omitempty"` + StartCursor string `json:"startCursor,omitempty"` + } `json:"pageInfo,omitempty"` + Edges []struct { + Node struct { + ID string `json:"id,omitempty"` + ExternalIdentifier string `json:"externalIdentifier,omitempty"` + ExternalReference string `json:"externalReference,omitempty"` + FixedIn string `json:"fixedIn,omitempty"` + AffectedRange string `json:"affectedRange,omitempty"` + PackageName string `json:"packageName,omitempty"` + DismissReason string `json:"dismissReason,omitempty"` + DismissedAt time.Time `json:"dismissedAt,omitempty"` + } `json:"node,omitempty"` + } `json:"edges,omitempty"` + } `json:"vulnerabilityAlerts,omitempty"` + } `json:"repository,omitempty"` + } `json:"data,omitempty"` +} diff --git a/models/vulninfos.go b/models/vulninfos.go index 54ff63b2..b0c23d23 100644 --- a/models/vulninfos.go +++ b/models/vulninfos.go @@ -163,14 +163,47 @@ type PackageStatus struct { // VulnInfo has a vulnerability information and unsecure packages type VulnInfo struct { - CveID string `json:"cveID"` - Confidences Confidences `json:"confidences"` - AffectedPackages PackageStatuses `json:"affectedPackages"` + CveID string `json:"cveID,omitempty"` + Confidences Confidences `json:"confidences,omitempty"` + AffectedPackages PackageStatuses `json:"affectedPackages,omitempty"` DistroAdvisories []DistroAdvisory `json:"distroAdvisories,omitempty"` // for Aamazon, RHEL, FreeBSD - CpeURIs []string `json:"cpeURIs,omitempty"` // CpeURIs related to this CVE defined in config.toml - CveContents CveContents `json:"cveContents"` - Exploits []Exploit `json:"exploits"` - AlertDict AlertDict `json:"alertDict"` + CveContents CveContents `json:"cveContents,omitempty"` + Exploits []Exploit `json:"exploits,omitempty"` + AlertDict AlertDict `json:"alertDict,omitempty"` + + CpeURIs []string `json:"cpeURIs,omitempty"` // CpeURIs related to this CVE defined in config.toml + GitHubSecurityAlerts GitHubSecurityAlerts `json:"gitHubSecurityAlerts,omitempty"` +} + +// GitHubSecurityAlerts is a list of GitHubSecurityAlert +type GitHubSecurityAlerts []GitHubSecurityAlert + +// Add adds given arg to the slice and return the slice (imutable) +func (g GitHubSecurityAlerts) Add(alert GitHubSecurityAlert) GitHubSecurityAlerts { + for _, a := range g { + if a.PackageName == alert.PackageName { + return g + } + } + return append(g, alert) +} + +func (g GitHubSecurityAlerts) String() string { + ss := []string{} + for _, a := range g { + ss = append(ss, a.PackageName) + } + return strings.Join(ss, ", ") +} + +// GitHubSecurityAlert has detected CVE-ID, PackageName, Status fetched via GitHub API +type GitHubSecurityAlert struct { + PackageName string `json:"packageName"` + FixedIn string `json:"fixedIn"` + AffectedRange string `json:"affectedRange"` + Dismissed bool `json:"dismissed"` + DismissedAt time.Time `json:"dismissedAt"` + DismissReason string `json:"dismissReason"` } // Titles returns tilte (TUI) @@ -774,6 +807,9 @@ const ( // ChangelogLenientMatchStr is a String representation of ChangelogLenientMatch ChangelogLenientMatchStr = "ChangelogLenientMatch" + // GitHubMatchStr is a String representation of GitHubMatch + GitHubMatchStr = "GitHubMatch" + // FailedToGetChangelog is a String representation of FailedToGetChangelog FailedToGetChangelog = "FailedToGetChangelog" @@ -805,4 +841,7 @@ var ( // ChangelogLenientMatch is a ranking how confident the CVE-ID was deteted correctly ChangelogLenientMatch = Confidence{50, ChangelogLenientMatchStr, 4} + + // GitHubMatch is a ranking how confident the CVE-ID was deteted correctly + GitHubMatch = Confidence{97, GitHubMatchStr, 2} ) diff --git a/report/report.go b/report/report.go index 4d6a0ecb..269c6139 100644 --- a/report/report.go +++ b/report/report.go @@ -33,6 +33,7 @@ import ( "github.com/future-architect/vuls/contrib/owasp-dependency-check/parser" "github.com/future-architect/vuls/cwe" "github.com/future-architect/vuls/exploit" + "github.com/future-architect/vuls/github" "github.com/future-architect/vuls/gost" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/oval" @@ -144,7 +145,7 @@ func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]mode } // FillCveInfo fill scanResult with cve info. -func FillCveInfo(dbclient DBClient, r *models.ScanResult, cpeURIs []string) error { +func FillCveInfo(dbclient DBClient, r *models.ScanResult, cpeURIs []string, integrations ...c.IntegrationConf) error { util.Log.Debugf("need to refresh") nCVEs, err := FillWithOval(dbclient.OvalDB, r) @@ -169,6 +170,17 @@ func FillCveInfo(dbclient DBClient, r *models.ScanResult, cpeURIs []string) erro } util.Log.Infof("%s: %d CVEs are detected with CPE", r.FormatServerName(), nCVEs) + if len(integrations) != 0 { + for k, v := range integrations[0].GitHubConf { + c.Conf.Servers[r.ServerName].GitHubRepos[k] = v + } + } + nCVEs, err = fillGitHubSecurityAlerts(r) + if err != nil { + return fmt.Errorf("Failed to access GitHub Security Alerts: %s", err) + } + util.Log.Infof("%s: %d CVEs are detected with GitHub Security Alerts", r.FormatServerName(), nCVEs) + nCVEs, err = FillWithGost(dbclient.GostDB, r) if err != nil { return fmt.Errorf("Failed to fill with gost: %s", err) @@ -344,6 +356,21 @@ func fillVulnByCpeURIs(driver cvedb.DB, r *models.ScanResult, cpeURIs []string) return nCVEs, nil } +// https://help.github.com/articles/about-security-alerts-for-vulnerable-dependencies/ +func fillGitHubSecurityAlerts(r *models.ScanResult) (nCVEs int, err error) { + repos := c.Conf.Servers[r.ServerName].GitHubRepos + for ownerRepo, setting := range repos { + ss := strings.Split(ownerRepo, "/") + owner, repo := ss[0], ss[1] + n, err := github.FillGitHubSecurityAlerts(r, owner, repo, setting.Token) + if err != nil { + return 0, err + } + nCVEs += n + } + return nCVEs, nil +} + func fillCweDict(r *models.ScanResult) { uniqCweIDMap := map[string]bool{} for _, vinfo := range r.ScannedCves { diff --git a/report/slack.go b/report/slack.go index 4fb73b5c..2f498fe1 100644 --- a/report/slack.go +++ b/report/slack.go @@ -207,6 +207,9 @@ func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) { for _, n := range vinfo.CpeURIs { curent = append(curent, n) } + for _, n := range vinfo.GitHubSecurityAlerts { + curent = append(curent, n.PackageName) + } new := []string{} for _, affected := range vinfo.AffectedPackages { @@ -223,6 +226,9 @@ func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) { for range vinfo.CpeURIs { new = append(new, "?") } + for range vinfo.GitHubSecurityAlerts { + new = append(new, "?") + } a := slack.Attachment{ Title: vinfo.CveID, diff --git a/report/tui.go b/report/tui.go index 10e496e6..c7b6f5d1 100644 --- a/report/tui.go +++ b/report/tui.go @@ -636,6 +636,7 @@ func summaryLines(r models.ScanResult) string { packname := vinfo.AffectedPackages.FormatTuiSummary() packname += strings.Join(vinfo.CpeURIs, ", ") + packname += vinfo.GitHubSecurityAlerts.String() alert := " " if vinfo.AlertDict.HasAlert() { @@ -742,6 +743,10 @@ func setChangelogLayout(g *gocui.Gui) error { lines = append(lines, "* "+uri) } + for _, alert := range vinfo.GitHubSecurityAlerts { + lines = append(lines, "* "+alert.PackageName) + } + for _, adv := range vinfo.DistroAdvisories { lines = append(lines, "\n", "Advisories", diff --git a/report/util.go b/report/util.go index c09aed58..aaab0649 100644 --- a/report/util.go +++ b/report/util.go @@ -239,6 +239,10 @@ No CVE-IDs are found in updatable packages. data = append(data, []string{"CPE", name}) } + for _, alert := range vuln.GitHubSecurityAlerts { + data = append(data, []string{"GitHub", alert.PackageName}) + } + for _, confidence := range vuln.Confidences { data = append(data, []string{"Confidence", confidence.String()}) }