feat(report): GitHub security alerts integration (#775)

feat(report): integrate to GitHub security alerts
This commit is contained in:
Kota Kanbe
2019-02-20 12:04:10 +09:00
committed by GitHub
parent 256c99ffa2
commit 56d7d43768
10 changed files with 340 additions and 57 deletions

28
Gopkg.lock generated
View File

@@ -733,10 +733,11 @@
[[projects]]
branch = "master"
digest = "1:35d68717797810014f26d6a175b0e73bd7fba070c408d443b3fff87a61c8008e"
digest = "1:25c965216c188afcd5f430f65acc28d4f193c5a88e7c6c54ae876b1898f86368"
name = "golang.org/x/net"
packages = [
"context",
"context/ctxhttp",
"http/httpguts",
"http2",
"http2/hpack",
@@ -748,6 +749,17 @@
pruneopts = "UT"
revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc"
[[projects]]
branch = "master"
digest = "1:e007b54f54cbd4214aa6d97a67d57bc2539991adb4e22ea92c482bbece8de469"
name = "golang.org/x/oauth2"
packages = [
".",
"internal",
]
pruneopts = "UT"
revision = "99b60b757ec124ebb7d6b7e97f153b19c10ce163"
[[projects]]
branch = "master"
digest = "1:75515eedc0dc2cb0b40372008b616fa2841d831c63eedd403285ff286c593295"
@@ -799,9 +811,18 @@
version = "v0.1.0"
[[projects]]
digest = "1:c25289f43ac4a68d88b02245742347c94f1e108c534dda442188015ff80669b3"
digest = "1:9e29a0ec029d012437d88da3ccccf18adcdce069cab08d462056c2c6bb006505"
name = "google.golang.org/appengine"
packages = ["cloudsql"]
packages = [
"cloudsql",
"internal",
"internal/base",
"internal/datastore",
"internal/log",
"internal/remote_api",
"internal/urlfetch",
"urlfetch",
]
pruneopts = "UT"
revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1"
version = "v1.4.0"
@@ -964,6 +985,7 @@
"github.com/sirupsen/logrus",
"golang.org/x/crypto/ssh",
"golang.org/x/crypto/ssh/agent",
"golang.org/x/oauth2",
]
solver-name = "gps-cdcl"
solver-version = 1

View File

@@ -64,6 +64,7 @@ func (*ReportCmd) Usage() string {
[-diff]
[-ignore-unscored-cves]
[-ignore-unfixed]
[-ignore-github-dismissed]
[-to-email]
[-to-http]
[-to-slack]
@@ -133,10 +134,12 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) {
f.BoolVar(&c.Conf.IgnoreUnscoredCves, "ignore-unscored-cves", false,
"Don't report the unscored CVEs")
f.BoolVar(
&c.Conf.IgnoreUnfixed, "ignore-unfixed", false,
f.BoolVar(&c.Conf.IgnoreUnfixed, "ignore-unfixed", false,
"Don't report the unfixed CVEs")
f.BoolVar(&c.Conf.IgnoreGitHubDismissed, "ignore-github-dismissed", false,
"Don't report the dismissed CVEs on GitHub Security Alerts")
f.StringVar(
&c.Conf.HTTPProxy, "http-proxy", "",
"http://proxy-url:port (default: empty)")

View File

@@ -98,31 +98,34 @@ const (
//Config is struct of Configuration
type Config struct {
Debug bool `json:"debug"`
DebugSQL bool `json:"debugSQL"`
Lang string `json:"lang"`
HTTPProxy string `valid:"url" json:"httpProxy"`
LogDir string `json:"logDir"`
ResultsDir string `json:"resultsDir"`
Pipe bool `json:"pipe"`
Debug bool `json:"debug,omitempty"`
DebugSQL bool `json:"debugSQL,omitempty"`
Lang string `json:"lang,omitempty"`
HTTPProxy string `valid:"url" json:"httpProxy,omitempty"`
LogDir string `json:"logDir,omitempty"`
ResultsDir string `json:"resultsDir,omitempty"`
Pipe bool `json:"pipe,omitempty"`
Default ServerInfo `json:"default"`
Servers map[string]ServerInfo `json:"servers"`
CvssScoreOver float64 `json:"cvssScoreOver"`
IgnoreUnscoredCves bool `json:"ignoreUnscoredCves"`
IgnoreUnfixed bool `json:"ignoreUnfixed"`
SSHNative bool `json:"sshNative"`
SSHConfig bool `json:"sshConfig"`
ContainersOnly bool `json:"containersOnly"`
SkipBroken bool `json:"skipBroken"`
CacheDBPath string `json:"cacheDBPath"`
Vvv bool `json:"vvv"`
UUID bool `json:"uuid"`
Default ServerInfo `json:"default,omitempty"`
Servers map[string]ServerInfo `json:"servers,omitempty"`
CvssScoreOver float64 `json:"cvssScoreOver,omitempty"`
CveDict GoCveDictConf `json:"cveDict"`
OvalDict GovalDictConf `json:"ovalDict"`
Gost GostConf `json:"gost"`
Exploit ExploitConf `json:"exploit"`
IgnoreUnscoredCves bool `json:"ignoreUnscoredCves,omitempty"`
IgnoreUnfixed bool `json:"ignoreUnfixed,omitempty"`
IgnoreGitHubDismissed bool `json:"ignore_git_hub_dismissed,omitempty"`
SSHNative bool `json:"sshNative,omitempty"`
SSHConfig bool `json:"sshConfig,omitempty"`
ContainersOnly bool `json:"containersOnly,omitempty"`
SkipBroken bool `json:"skipBroken,omitempty"`
CacheDBPath string `json:"cacheDBPath,omitempty"`
Vvv bool `json:"vvv,omitempty"`
UUID bool `json:"uuid,omitempty"`
CveDict GoCveDictConf `json:"cveDict,omitempty"`
OvalDict GovalDictConf `json:"ovalDict,omitempty"`
Gost GostConf `json:"gost,omitempty"`
Exploit ExploitConf `json:"exploit,omitempty"`
Slack SlackConf `json:"-"`
EMail SMTPConf `json:"-"`
@@ -136,27 +139,27 @@ type Config struct {
Telegram TelegramConf `json:"-"`
Saas SaasConf `json:"-"`
RefreshCve bool `json:"refreshCve"`
ToSlack bool `json:"toSlack"`
ToStride bool `json:"toStride"`
ToHipChat bool `json:"toHipChat"`
ToChatWork bool `json:"toChatWork"`
ToTelegram bool `json:"ToTelegram"`
ToEmail bool `json:"toEmail"`
ToSyslog bool `json:"toSyslog"`
ToLocalFile bool `json:"toLocalFile"`
ToS3 bool `json:"toS3"`
ToAzureBlob bool `json:"toAzureBlob"`
ToSaas bool `json:"toSaas"`
ToHTTP bool `json:"toHTTP"`
FormatXML bool `json:"formatXML"`
FormatJSON bool `json:"formatJSON"`
FormatOneEMail bool `json:"formatOneEMail"`
FormatOneLineText bool `json:"formatOneLineText"`
FormatList bool `json:"formatList"`
FormatFullText bool `json:"formatFullText"`
GZIP bool `json:"gzip"`
Diff bool `json:"diff"`
RefreshCve bool `json:"refreshCve,omitempty"`
ToSlack bool `json:"toSlack,omitempty"`
ToStride bool `json:"toStride,omitempty"`
ToHipChat bool `json:"toHipChat,omitempty"`
ToChatWork bool `json:"toChatWork,omitempty"`
ToTelegram bool `json:"ToTelegram,omitempty"`
ToEmail bool `json:"toEmail,omitempty"`
ToSyslog bool `json:"toSyslog,omitempty"`
ToLocalFile bool `json:"toLocalFile,omitempty"`
ToS3 bool `json:"toS3,omitempty"`
ToAzureBlob bool `json:"toAzureBlob,omitempty"`
ToSaas bool `json:"toSaas,omitempty"`
ToHTTP bool `json:"toHTTP,omitempty"`
FormatXML bool `json:"formatXML,omitempty"`
FormatJSON bool `json:"formatJSON,omitempty"`
FormatOneEMail bool `json:"formatOneEMail,omitempty"`
FormatOneLineText bool `json:"formatOneLineText,omitempty"`
FormatList bool `json:"formatList,omitempty"`
FormatFullText bool `json:"formatFullText,omitempty"`
GZIP bool `json:"gzip,omitempty"`
Diff bool `json:"diff,omitempty"`
}
// ValidateOnConfigtest validates
@@ -1054,6 +1057,7 @@ type ServerInfo struct {
Containers map[string]ContainerSetting `toml:"containers" json:"containers,omitempty"`
IgnoreCves []string `toml:"ignoreCves,omitempty" json:"ignoreCves,omitempty"`
IgnorePkgsRegexp []string `toml:"ignorePkgsRegexp,omitempty" json:"ignorePkgsRegexp,omitempty"`
GitHubRepos map[string]GitHubConf `toml:"githubs" json:"githubs,omitempty"` // key: owner/repo
UUIDs map[string]string `toml:"uuids,omitempty" json:"uuids,omitempty"`
Memo string `toml:"memo,omitempty" json:"memo"`
Enablerepo []string `toml:"enablerepo,omitempty" json:"enablerepo,omitempty"` // For CentOS, RHEL, Amazon
@@ -1077,6 +1081,23 @@ type ContainerSetting struct {
IgnoreCves []string `json:"ignoreCves,omitempty"`
}
// IntegrationConf is used for integration configuration
type IntegrationConf struct {
GitHubConf map[string]GitHubConf
}
// New creates IntegrationConf and initialize fields
func (c IntegrationConf) New() IntegrationConf {
return IntegrationConf{
GitHubConf: map[string]GitHubConf{},
}
}
// GitHubConf is used for GitHub integration
type GitHubConf struct {
Token string `json:"token"`
}
// ScanMode has a type of scan mode. fast, fast-root, deep and offline
type ScanMode struct {
flag byte

View File

@@ -254,6 +254,18 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error {
}
}
s.GitHubRepos = v.GitHubRepos
for ownerRepo, githubSetting := range s.GitHubRepos {
if ss := strings.Split(ownerRepo, "/"); len(ss) != 2 {
return fmt.Errorf("Failed to parse GitHub owner/repo: %s in %s",
ownerRepo, serverName)
}
if githubSetting.Token == "" {
return fmt.Errorf("GitHub owner/repo: %s in %s token is empty",
ownerRepo, serverName)
}
}
s.UUIDs = v.UUIDs
s.Type = v.Type

144
github/github.go Normal file
View File

@@ -0,0 +1,144 @@
/* Vuls - Vulnerability Scanner
Copyright (C) 2016 Future Corporation , Japan.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package github
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
"github.com/k0kubun/pp"
"golang.org/x/oauth2"
)
// FillGitHubSecurityAlerts access to owner/repo on GitHub and fetch scurity alerts of the repository via GitHub API v4 GraphQL and then set to the given ScanResult.
// https://help.github.com/articles/about-security-alerts-for-vulnerable-dependencies/
func FillGitHubSecurityAlerts(r *models.ScanResult, owner, repo, token string) (nCVEs int, err error) {
src := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
)
httpClient := oauth2.NewClient(context.Background(), src)
// TODO Use `https://github.com/shurcooL/githubv4` if the tool supports vulnerabilityAlerts Endpoint
const jsonfmt = `{"query":
"query { repository(owner:\"%s\", name:\"%s\") { url, vulnerabilityAlerts(first: %d, %s) { pageInfo{ endCursor, hasNextPage, startCursor}, edges { node { id, externalIdentifier, externalReference, fixedIn, packageName, dismissReason, dismissedAt } } } } }"}`
after := ""
for {
jsonStr := fmt.Sprintf(jsonfmt, owner, repo, 100, after)
req, err := http.NewRequest("POST",
"https://api.github.com/graphql",
bytes.NewBuffer([]byte(jsonStr)),
)
if err != nil {
return 0, err
}
// https://developer.github.com/v4/previews/#repository-vulnerability-alerts
// To toggle this preview and access data, need to provide a custom media type in the Accept header:
// MEMO: I tried to get the affected version via GitHub API. Bit it seems difficult to determin the affected version if there are multiple dependency files such as package.json.
// TODO remove this header if it is no longer preview status in the future.
req.Header.Set("Accept", "application/vnd.github.vixen-preview+json")
req.Header.Set("Content-Type", "application/json")
resp, err := httpClient.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
alerts := SecurityAlerts{}
if json.NewDecoder(resp.Body).Decode(&alerts); err != nil {
return 0, err
}
util.Log.Debugf("%s", pp.Sprint(alerts))
for _, v := range alerts.Data.Repository.VulnerabilityAlerts.Edges {
if config.Conf.IgnoreGitHubDismissed && v.Node.DismissReason != "" {
continue
}
pkgName := fmt.Sprintf("%s %s",
alerts.Data.Repository.URL, v.Node.PackageName)
m := models.GitHubSecurityAlert{
PackageName: pkgName,
FixedIn: v.Node.FixedIn,
AffectedRange: v.Node.AffectedRange,
Dismissed: len(v.Node.DismissReason) != 0,
DismissedAt: v.Node.DismissedAt,
DismissReason: v.Node.DismissReason,
}
cveID := v.Node.ExternalIdentifier
if val, ok := r.ScannedCves[cveID]; ok {
val.GitHubSecurityAlerts = val.GitHubSecurityAlerts.Add(m)
r.ScannedCves[cveID] = val
nCVEs++
} else {
v := models.VulnInfo{
CveID: cveID,
Confidences: models.Confidences{models.GitHubMatch},
GitHubSecurityAlerts: models.GitHubSecurityAlerts{m},
}
r.ScannedCves[cveID] = v
nCVEs++
}
}
if !alerts.Data.Repository.VulnerabilityAlerts.PageInfo.HasNextPage {
break
}
after = fmt.Sprintf(`after: \"%s\"`, alerts.Data.Repository.VulnerabilityAlerts.PageInfo.EndCursor)
}
return nCVEs, err
}
//SecurityAlerts has detected CVE-IDs, PackageNames, Refs
type SecurityAlerts struct {
Data struct {
Repository struct {
URL string `json:"url,omitempty"`
VulnerabilityAlerts struct {
PageInfo struct {
EndCursor string `json:"endCursor,omitempty"`
HasNextPage bool `json:"hasNextPage,omitempty"`
StartCursor string `json:"startCursor,omitempty"`
} `json:"pageInfo,omitempty"`
Edges []struct {
Node struct {
ID string `json:"id,omitempty"`
ExternalIdentifier string `json:"externalIdentifier,omitempty"`
ExternalReference string `json:"externalReference,omitempty"`
FixedIn string `json:"fixedIn,omitempty"`
AffectedRange string `json:"affectedRange,omitempty"`
PackageName string `json:"packageName,omitempty"`
DismissReason string `json:"dismissReason,omitempty"`
DismissedAt time.Time `json:"dismissedAt,omitempty"`
} `json:"node,omitempty"`
} `json:"edges,omitempty"`
} `json:"vulnerabilityAlerts,omitempty"`
} `json:"repository,omitempty"`
} `json:"data,omitempty"`
}

View File

@@ -163,14 +163,47 @@ type PackageStatus struct {
// VulnInfo has a vulnerability information and unsecure packages
type VulnInfo struct {
CveID string `json:"cveID"`
Confidences Confidences `json:"confidences"`
AffectedPackages PackageStatuses `json:"affectedPackages"`
CveID string `json:"cveID,omitempty"`
Confidences Confidences `json:"confidences,omitempty"`
AffectedPackages PackageStatuses `json:"affectedPackages,omitempty"`
DistroAdvisories []DistroAdvisory `json:"distroAdvisories,omitempty"` // for Aamazon, RHEL, FreeBSD
CpeURIs []string `json:"cpeURIs,omitempty"` // CpeURIs related to this CVE defined in config.toml
CveContents CveContents `json:"cveContents"`
Exploits []Exploit `json:"exploits"`
AlertDict AlertDict `json:"alertDict"`
CveContents CveContents `json:"cveContents,omitempty"`
Exploits []Exploit `json:"exploits,omitempty"`
AlertDict AlertDict `json:"alertDict,omitempty"`
CpeURIs []string `json:"cpeURIs,omitempty"` // CpeURIs related to this CVE defined in config.toml
GitHubSecurityAlerts GitHubSecurityAlerts `json:"gitHubSecurityAlerts,omitempty"`
}
// GitHubSecurityAlerts is a list of GitHubSecurityAlert
type GitHubSecurityAlerts []GitHubSecurityAlert
// Add adds given arg to the slice and return the slice (imutable)
func (g GitHubSecurityAlerts) Add(alert GitHubSecurityAlert) GitHubSecurityAlerts {
for _, a := range g {
if a.PackageName == alert.PackageName {
return g
}
}
return append(g, alert)
}
func (g GitHubSecurityAlerts) String() string {
ss := []string{}
for _, a := range g {
ss = append(ss, a.PackageName)
}
return strings.Join(ss, ", ")
}
// GitHubSecurityAlert has detected CVE-ID, PackageName, Status fetched via GitHub API
type GitHubSecurityAlert struct {
PackageName string `json:"packageName"`
FixedIn string `json:"fixedIn"`
AffectedRange string `json:"affectedRange"`
Dismissed bool `json:"dismissed"`
DismissedAt time.Time `json:"dismissedAt"`
DismissReason string `json:"dismissReason"`
}
// Titles returns tilte (TUI)
@@ -774,6 +807,9 @@ const (
// ChangelogLenientMatchStr is a String representation of ChangelogLenientMatch
ChangelogLenientMatchStr = "ChangelogLenientMatch"
// GitHubMatchStr is a String representation of GitHubMatch
GitHubMatchStr = "GitHubMatch"
// FailedToGetChangelog is a String representation of FailedToGetChangelog
FailedToGetChangelog = "FailedToGetChangelog"
@@ -805,4 +841,7 @@ var (
// ChangelogLenientMatch is a ranking how confident the CVE-ID was deteted correctly
ChangelogLenientMatch = Confidence{50, ChangelogLenientMatchStr, 4}
// GitHubMatch is a ranking how confident the CVE-ID was deteted correctly
GitHubMatch = Confidence{97, GitHubMatchStr, 2}
)

View File

@@ -33,6 +33,7 @@ import (
"github.com/future-architect/vuls/contrib/owasp-dependency-check/parser"
"github.com/future-architect/vuls/cwe"
"github.com/future-architect/vuls/exploit"
"github.com/future-architect/vuls/github"
"github.com/future-architect/vuls/gost"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/oval"
@@ -144,7 +145,7 @@ func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]mode
}
// FillCveInfo fill scanResult with cve info.
func FillCveInfo(dbclient DBClient, r *models.ScanResult, cpeURIs []string) error {
func FillCveInfo(dbclient DBClient, r *models.ScanResult, cpeURIs []string, integrations ...c.IntegrationConf) error {
util.Log.Debugf("need to refresh")
nCVEs, err := FillWithOval(dbclient.OvalDB, r)
@@ -169,6 +170,17 @@ func FillCveInfo(dbclient DBClient, r *models.ScanResult, cpeURIs []string) erro
}
util.Log.Infof("%s: %d CVEs are detected with CPE", r.FormatServerName(), nCVEs)
if len(integrations) != 0 {
for k, v := range integrations[0].GitHubConf {
c.Conf.Servers[r.ServerName].GitHubRepos[k] = v
}
}
nCVEs, err = fillGitHubSecurityAlerts(r)
if err != nil {
return fmt.Errorf("Failed to access GitHub Security Alerts: %s", err)
}
util.Log.Infof("%s: %d CVEs are detected with GitHub Security Alerts", r.FormatServerName(), nCVEs)
nCVEs, err = FillWithGost(dbclient.GostDB, r)
if err != nil {
return fmt.Errorf("Failed to fill with gost: %s", err)
@@ -344,6 +356,21 @@ func fillVulnByCpeURIs(driver cvedb.DB, r *models.ScanResult, cpeURIs []string)
return nCVEs, nil
}
// https://help.github.com/articles/about-security-alerts-for-vulnerable-dependencies/
func fillGitHubSecurityAlerts(r *models.ScanResult) (nCVEs int, err error) {
repos := c.Conf.Servers[r.ServerName].GitHubRepos
for ownerRepo, setting := range repos {
ss := strings.Split(ownerRepo, "/")
owner, repo := ss[0], ss[1]
n, err := github.FillGitHubSecurityAlerts(r, owner, repo, setting.Token)
if err != nil {
return 0, err
}
nCVEs += n
}
return nCVEs, nil
}
func fillCweDict(r *models.ScanResult) {
uniqCweIDMap := map[string]bool{}
for _, vinfo := range r.ScannedCves {

View File

@@ -207,6 +207,9 @@ func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) {
for _, n := range vinfo.CpeURIs {
curent = append(curent, n)
}
for _, n := range vinfo.GitHubSecurityAlerts {
curent = append(curent, n.PackageName)
}
new := []string{}
for _, affected := range vinfo.AffectedPackages {
@@ -223,6 +226,9 @@ func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) {
for range vinfo.CpeURIs {
new = append(new, "?")
}
for range vinfo.GitHubSecurityAlerts {
new = append(new, "?")
}
a := slack.Attachment{
Title: vinfo.CveID,

View File

@@ -636,6 +636,7 @@ func summaryLines(r models.ScanResult) string {
packname := vinfo.AffectedPackages.FormatTuiSummary()
packname += strings.Join(vinfo.CpeURIs, ", ")
packname += vinfo.GitHubSecurityAlerts.String()
alert := " "
if vinfo.AlertDict.HasAlert() {
@@ -742,6 +743,10 @@ func setChangelogLayout(g *gocui.Gui) error {
lines = append(lines, "* "+uri)
}
for _, alert := range vinfo.GitHubSecurityAlerts {
lines = append(lines, "* "+alert.PackageName)
}
for _, adv := range vinfo.DistroAdvisories {
lines = append(lines, "\n",
"Advisories",

View File

@@ -239,6 +239,10 @@ No CVE-IDs are found in updatable packages.
data = append(data, []string{"CPE", name})
}
for _, alert := range vuln.GitHubSecurityAlerts {
data = append(data, []string{"GitHub", alert.PackageName})
}
for _, confidence := range vuln.Confidences {
data = append(data, []string{"Confidence", confidence.String()})
}