refactor: don't use global Config in private func (#1197)

* refactor: cve_client.go

* refactor: don't use global Config in private func

* remove import alias for config

* refactor: dbclient

* refactor: resultDir

* refactor: resultsDir

* refactor

* refactor: gost

* refactor: db client

* refactor: cveDB

* refactor: cvedb

* refactor: exploitDB

* refactor: remove detector/dbclient.go

* refactor: writer

* refactor: syslog writer

* refactor: ips

* refactor: ensureResultDir

* refactor: proxy

* fix(db): call CloseDB

* add integration test

* feat(report): sort array in json

* sort func for json diff

* add build-int to makefile

* add int-rds-redis to makefile

* fix: test case, makefile

* fix makefile

* show cve count after diff

* make diff

* diff -c

* sort exploits in json for diff

* sort metasploit, exploit
This commit is contained in:
Kota Kanbe
2021-04-01 13:36:24 +09:00
committed by GitHub
parent 0179f4299a
commit 9bfe0627ae
70 changed files with 48982 additions and 1274 deletions

View File

@@ -16,15 +16,50 @@ import (
"github.com/future-architect/vuls/logging"
"github.com/future-architect/vuls/util"
cvedb "github.com/kotakanbe/go-cve-dictionary/db"
cvelog "github.com/kotakanbe/go-cve-dictionary/log"
cvemodels "github.com/kotakanbe/go-cve-dictionary/models"
)
// CveClient is api client of CVE dictionary service.
var CveClient cvedictClient
type goCveDictClient struct {
cnf config.VulnDictInterface
driver cvedb.DB
}
type cvedictClient struct {
// httpProxy string
baseURL string
func newGoCveDictClient(cnf config.VulnDictInterface, o logging.LogOpts) (*goCveDictClient, error) {
cvelog.SetLogger(o.LogDir, o.Quiet, o.Debug, false)
driver, locked, err := newCveDB(cnf)
if locked {
return nil, xerrors.Errorf("SQLite3 is locked: %s", cnf.GetSQLite3Path())
} else if err != nil {
return nil, err
}
return &goCveDictClient{cnf: cnf, driver: driver}, nil
}
func (api goCveDictClient) closeDB() error {
if api.driver == nil {
return nil
}
if err := api.driver.CloseDB(); err != nil {
return xerrors.Errorf("Failed to close DB: %+v", err)
}
return nil
}
func (api goCveDictClient) fetchCveDetails(cveIDs []string) (cveDetails []cvemodels.CveDetail, err error) {
for _, cveID := range cveIDs {
cveDetail, err := api.driver.Get(cveID)
if err != nil {
return nil, xerrors.Errorf("Failed to fetch CVE. err: %w", err)
}
if len(cveDetail.CveID) == 0 {
cveDetails = append(cveDetails, cvemodels.CveDetail{CveID: cveID})
} else {
cveDetails = append(cveDetails, *cveDetail)
}
}
return
}
type response struct {
@@ -32,28 +67,7 @@ type response struct {
CveDetail cvemodels.CveDetail
}
func (api cvedictClient) FetchCveDetails(driver cvedb.DB, cveIDs []string) (cveDetails []cvemodels.CveDetail, err error) {
if !config.Conf.CveDict.IsFetchViaHTTP() {
if driver == nil {
return
}
for _, cveID := range cveIDs {
cveDetail, err := driver.Get(cveID)
if err != nil {
return nil, xerrors.Errorf("Failed to fetch CVE. err: %w", err)
}
if len(cveDetail.CveID) == 0 {
cveDetails = append(cveDetails, cvemodels.CveDetail{
CveID: cveID,
})
} else {
cveDetails = append(cveDetails, *cveDetail)
}
}
return
}
api.baseURL = config.Conf.CveDict.URL
func (api goCveDictClient) fetchCveDetailsViaHTTP(cveIDs []string) (cveDetails []cvemodels.CveDetail, err error) {
reqChan := make(chan string, len(cveIDs))
resChan := make(chan response, len(cveIDs))
errChan := make(chan error, len(cveIDs))
@@ -73,7 +87,7 @@ func (api cvedictClient) FetchCveDetails(driver cvedb.DB, cveIDs []string) (cveD
tasks <- func() {
select {
case cveID := <-reqChan:
url, err := util.URLPathJoin(api.baseURL, "cves", cveID)
url, err := util.URLPathJoin(api.cnf.GetURL(), "cves", cveID)
if err != nil {
errChan <- err
} else {
@@ -109,12 +123,11 @@ func (api cvedictClient) FetchCveDetails(driver cvedb.DB, cveIDs []string) (cveD
return
}
func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errChan chan<- error) {
func (api goCveDictClient) httpGet(key, url string, resChan chan<- response, errChan chan<- error) {
var body string
var errs []error
var resp *http.Response
f := func() (err error) {
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
return xerrors.Errorf("HTTP GET Error, url: %s, resp: %v, err: %+v",
@@ -123,8 +136,7 @@ func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errCh
return nil
}
notify := func(err error, t time.Duration) {
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v",
t, err)
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
}
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
if err != nil {
@@ -142,10 +154,9 @@ func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errCh
}
}
func (api cvedictClient) FetchCveDetailsByCpeName(driver cvedb.DB, cpeName string) ([]cvemodels.CveDetail, error) {
if config.Conf.CveDict.IsFetchViaHTTP() {
api.baseURL = config.Conf.CveDict.URL
url, err := util.URLPathJoin(api.baseURL, "cpes")
func (api goCveDictClient) fetchCveDetailsByCpeName(cpeName string) ([]cvemodels.CveDetail, error) {
if api.cnf.IsFetchViaHTTP() {
url, err := util.URLPathJoin(api.cnf.GetURL(), "cpes")
if err != nil {
return nil, err
}
@@ -154,15 +165,14 @@ func (api cvedictClient) FetchCveDetailsByCpeName(driver cvedb.DB, cpeName strin
logging.Log.Debugf("HTTP Request to %s, query: %#v", url, query)
return api.httpPost(cpeName, url, query)
}
return driver.GetByCpeURI(cpeName)
return api.driver.GetByCpeURI(cpeName)
}
func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]cvemodels.CveDetail, error) {
func (api goCveDictClient) httpPost(key, url string, query map[string]string) ([]cvemodels.CveDetail, error) {
var body string
var errs []error
var resp *http.Response
f := func() (err error) {
// req := gorequest.New().SetDebug(config.Conf.Debug).Post(url)
req := gorequest.New().Timeout(10 * time.Second).Post(url)
for key := range query {
req = req.Send(fmt.Sprintf("%s=%s", key, query[key])).Type("json")
@@ -188,3 +198,19 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c
}
return cveDetails, nil
}
func newCveDB(cnf config.VulnDictInterface) (driver cvedb.DB, locked bool, err error) {
if cnf.IsFetchViaHTTP() {
return nil, false, nil
}
path := cnf.GetURL()
if cnf.GetType() == "sqlite3" {
path = cnf.GetSQLite3Path()
}
driver, locked, err = cvedb.NewDB(cnf.GetType(), path, cnf.GetDebugSQL())
if err != nil {
err = xerrors.Errorf("Failed to init CVE DB. err: %w, path: %s", err, path)
return nil, locked, err
}
return driver, false, nil
}

View File

@@ -1,184 +0,0 @@
// +build !scanner
package detector
import (
"github.com/future-architect/vuls/config"
gostdb "github.com/knqyf263/gost/db"
cvedb "github.com/kotakanbe/go-cve-dictionary/db"
ovaldb "github.com/kotakanbe/goval-dictionary/db"
metasploitdb "github.com/takuzoo3868/go-msfdb/db"
exploitdb "github.com/vulsio/go-exploitdb/db"
"golang.org/x/xerrors"
)
// DBClient is DB client for reporting
type DBClient struct {
CveDB cvedb.DB
OvalDB ovaldb.DB
GostDB gostdb.DB
ExploitDB exploitdb.DB
MetasploitDB metasploitdb.DB
}
// NewDBClient returns db clients
func NewDBClient(cveDict, ovalDict, gost, exploit, metasploit config.VulnDictInterface, debugSQL bool) (dbclient *DBClient, err error) {
for _, cnf := range []config.VulnDictInterface{cveDict, ovalDict, gost, exploit, metasploit} {
if err := cnf.Validate(); err != nil {
return nil, xerrors.Errorf("Failed to validate %s: %+v", cnf.GetName(), err)
}
if err := cnf.CheckHTTPHealth(); err != nil {
return nil, xerrors.Errorf("Run %s as server mode before reporting: %+v", cnf.GetName(), err)
}
}
cveDriver, locked, err := NewCveDB(cveDict, debugSQL)
if locked {
return nil, xerrors.Errorf("SQLite3 is locked: %s", cveDict.GetSQLite3Path())
} else if err != nil {
return nil, err
}
ovaldb, locked, err := NewOvalDB(ovalDict, debugSQL)
if locked {
return nil, xerrors.Errorf("SQLite3 is locked: %s", ovalDict.GetSQLite3Path())
} else if err != nil {
return nil, err
}
gostdb, locked, err := NewGostDB(gost, debugSQL)
if locked {
return nil, xerrors.Errorf("SQLite3 is locked: %s", gost.GetSQLite3Path())
} else if err != nil {
return nil, err
}
exploitdb, locked, err := NewExploitDB(exploit, debugSQL)
if locked {
return nil, xerrors.Errorf("SQLite3 is locked: %s", exploit.GetSQLite3Path())
} else if err != nil {
return nil, err
}
metasploitdb, locked, err := NewMetasploitDB(metasploit, debugSQL)
if locked {
return nil, xerrors.Errorf("SQLite3 is locked: %s", metasploit.GetSQLite3Path())
} else if err != nil {
return nil, err
}
return &DBClient{
CveDB: cveDriver,
OvalDB: ovaldb,
GostDB: gostdb,
ExploitDB: exploitdb,
MetasploitDB: metasploitdb,
}, nil
}
// NewCveDB returns cve db client
func NewCveDB(cnf config.VulnDictInterface, debugSQL bool) (driver cvedb.DB, locked bool, err error) {
if cnf.IsFetchViaHTTP() {
return nil, false, nil
}
path := cnf.GetURL()
if cnf.GetType() == "sqlite3" {
path = cnf.GetSQLite3Path()
}
driver, locked, err = cvedb.NewDB(cnf.GetType(), path, debugSQL)
if err != nil {
err = xerrors.Errorf("Failed to init CVE DB. err: %w, path: %s", err, path)
return nil, locked, err
}
return driver, false, nil
}
// NewOvalDB returns oval db client
func NewOvalDB(cnf config.VulnDictInterface, debugSQL bool) (driver ovaldb.DB, locked bool, err error) {
if cnf.IsFetchViaHTTP() {
return nil, false, nil
}
path := cnf.GetURL()
if cnf.GetType() == "sqlite3" {
path = cnf.GetSQLite3Path()
}
driver, locked, err = ovaldb.NewDB("", cnf.GetType(), path, debugSQL)
if err != nil {
err = xerrors.Errorf("Failed to new OVAL DB. err: %w", err)
if locked {
return nil, true, err
}
return nil, false, err
}
return driver, false, nil
}
// NewGostDB returns db client for Gost
func NewGostDB(cnf config.VulnDictInterface, debugSQL bool) (driver gostdb.DB, locked bool, err error) {
if cnf.IsFetchViaHTTP() {
return nil, false, nil
}
path := cnf.GetURL()
if cnf.GetType() == "sqlite3" {
path = cnf.GetSQLite3Path()
}
if driver, locked, err = gostdb.NewDB(cnf.GetType(), path, debugSQL); err != nil {
if locked {
return nil, true, xerrors.Errorf("gostDB is locked. err: %w", err)
}
return nil, false, err
}
return driver, false, nil
}
// NewExploitDB returns db client for Exploit
func NewExploitDB(cnf config.VulnDictInterface, debugSQL bool) (driver exploitdb.DB, locked bool, err error) {
if cnf.IsFetchViaHTTP() {
return nil, false, nil
}
path := cnf.GetURL()
if cnf.GetType() == "sqlite3" {
path = cnf.GetSQLite3Path()
}
if driver, locked, err = exploitdb.NewDB(cnf.GetType(), path, debugSQL); err != nil {
if locked {
return nil, true, xerrors.Errorf("exploitDB is locked. err: %w", err)
}
return nil, false, err
}
return driver, false, nil
}
// NewMetasploitDB returns db client for Metasploit
func NewMetasploitDB(cnf config.VulnDictInterface, debugSQL bool) (driver metasploitdb.DB, locked bool, err error) {
if cnf.IsFetchViaHTTP() {
return nil, false, nil
}
path := cnf.GetURL()
if cnf.GetType() == "sqlite3" {
path = cnf.GetSQLite3Path()
}
if driver, locked, err = metasploitdb.NewDB(cnf.GetType(), path, debugSQL, false); err != nil {
if locked {
return nil, true, xerrors.Errorf("metasploitDB is locked. err: %w", err)
}
return nil, false, err
}
return driver, false, nil
}
// CloseDB close dbs
func (d DBClient) CloseDB() (errs []error) {
if d.CveDB != nil {
if err := d.CveDB.CloseDB(); err != nil {
errs = append(errs, xerrors.Errorf("Failed to close cveDB. err: %+v", err))
}
}
if d.OvalDB != nil {
if err := d.OvalDB.CloseDB(); err != nil {
errs = append(errs, xerrors.Errorf("Failed to close ovalDB. err: %+v", err))
}
}
//TODO CloseDB gost, exploitdb, metasploit
return errs
}

View File

@@ -7,38 +7,27 @@ import (
"strings"
"time"
c "github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/constant"
"github.com/future-architect/vuls/contrib/owasp-dependency-check/parser"
"github.com/future-architect/vuls/cwe"
"github.com/future-architect/vuls/exploit"
"github.com/future-architect/vuls/gost"
"github.com/future-architect/vuls/logging"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/msf"
"github.com/future-architect/vuls/oval"
"github.com/future-architect/vuls/reporter"
"github.com/future-architect/vuls/util"
gostdb "github.com/knqyf263/gost/db"
cvedb "github.com/kotakanbe/go-cve-dictionary/db"
cvemodels "github.com/kotakanbe/go-cve-dictionary/models"
ovaldb "github.com/kotakanbe/goval-dictionary/db"
metasploitdb "github.com/takuzoo3868/go-msfdb/db"
exploitdb "github.com/vulsio/go-exploitdb/db"
"golang.org/x/xerrors"
)
// type Detector struct {
// Targets map[string]c.ServerInfo
// }
// Detect vulns and fill CVE detailed information
func Detect(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
func Detect(rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
// Use the same reportedAt for all rs
reportedAt := time.Now()
for i, r := range rs {
if !c.Conf.RefreshCve && !needToRefreshCve(r) {
if !config.Conf.RefreshCve && !needToRefreshCve(r) {
logging.Log.Info("No need to refresh")
continue
}
@@ -49,10 +38,10 @@ func Detect(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.Sca
cpeURIs, owaspDCXMLPath := []string{}, ""
if len(r.Container.ContainerID) == 0 {
cpeURIs = c.Conf.Servers[r.ServerName].CpeNames
owaspDCXMLPath = c.Conf.Servers[r.ServerName].OwaspDCXMLPath
cpeURIs = config.Conf.Servers[r.ServerName].CpeNames
owaspDCXMLPath = config.Conf.Servers[r.ServerName].OwaspDCXMLPath
} else {
if s, ok := c.Conf.Servers[r.ServerName]; ok {
if s, ok := config.Conf.Servers[r.ServerName]; ok {
if con, ok := s.Containers[r.Container.Name]; ok {
cpeURIs = con.Cpes
owaspDCXMLPath = con.OwaspDCXMLPath
@@ -68,46 +57,66 @@ func Detect(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.Sca
cpeURIs = append(cpeURIs, cpes...)
}
if err := DetectLibsCves(&r, c.Conf.TrivyCacheDBDir, c.Conf.NoProgress); err != nil {
if err := DetectLibsCves(&r, config.Conf.TrivyCacheDBDir, config.Conf.NoProgress); err != nil {
return nil, xerrors.Errorf("Failed to fill with Library dependency: %w", err)
}
if err := DetectPkgCves(dbclient, &r); err != nil {
if err := DetectPkgCves(&r, config.Conf.OvalDict, config.Conf.Gost); err != nil {
return nil, xerrors.Errorf("Failed to detect Pkg CVE: %w", err)
}
if err := DetectCpeURIsCves(dbclient.CveDB, &r, cpeURIs); err != nil {
if err := DetectCpeURIsCves(&r, cpeURIs, config.Conf.CveDict, config.Conf.LogOpts); err != nil {
return nil, xerrors.Errorf("Failed to detect CVE of `%s`: %w", cpeURIs, err)
}
repos := c.Conf.Servers[r.ServerName].GitHubRepos
if err := DetectGitHubCves(&r, repos, c.Conf.IgnoreGitHubDismissed); err != nil {
repos := config.Conf.Servers[r.ServerName].GitHubRepos
if err := DetectGitHubCves(&r, repos, config.Conf.IgnoreGitHubDismissed); err != nil {
return nil, xerrors.Errorf("Failed to detect GitHub Cves: %w", err)
}
if err := DetectWordPressCves(&r, &c.Conf.WpScan); err != nil {
if err := DetectWordPressCves(&r, config.Conf.WpScan); err != nil {
return nil, xerrors.Errorf("Failed to detect WordPress Cves: %w", err)
}
if err := FillCveInfo(dbclient, &r); err != nil {
return nil, err
logging.Log.Infof("Fill CVE detailed with gost")
if err := gost.FillCVEsWithRedHat(&r, config.Conf.Gost); err != nil {
return nil, xerrors.Errorf("Failed to fill with gost: %w", err)
}
logging.Log.Infof("Fill CVE detailed with go-cve-dictionary")
if err := FillCvesWithNvdJvn(&r, config.Conf.CveDict, config.Conf.LogOpts); err != nil {
return nil, xerrors.Errorf("Failed to fill with CVE: %w", err)
}
nExploitCve, err := FillWithExploit(&r, config.Conf.Exploit)
if err != nil {
return nil, xerrors.Errorf("Failed to fill with exploit: %w", err)
}
logging.Log.Infof("%s: %d exploits are detected", r.FormatServerName(), nExploitCve)
nMetasploitCve, err := FillWithMetasploit(&r, config.Conf.Metasploit)
if err != nil {
return nil, xerrors.Errorf("Failed to fill with metasploit: %w", err)
}
logging.Log.Infof("%s: %d modules are detected", r.FormatServerName(), nMetasploitCve)
FillCweDict(&r)
r.ReportedBy, _ = os.Hostname()
r.Lang = c.Conf.Lang
r.Lang = config.Conf.Lang
r.ReportedAt = reportedAt
r.ReportedVersion = c.Version
r.ReportedRevision = c.Revision
r.Config.Report = c.Conf
r.Config.Report.Servers = map[string]c.ServerInfo{
r.ServerName: c.Conf.Servers[r.ServerName],
r.ReportedVersion = config.Version
r.ReportedRevision = config.Revision
r.Config.Report = config.Conf
r.Config.Report.Servers = map[string]config.ServerInfo{
r.ServerName: config.Conf.Servers[r.ServerName],
}
rs[i] = r
}
// Overwrite the json file every time to clear the fields specified in config.IgnoredJSONKeys
for _, r := range rs {
if s, ok := c.Conf.Servers[r.ServerName]; ok {
if s, ok := config.Conf.Servers[r.ServerName]; ok {
r = r.ClearFields(s.IgnoredJSONKeys)
}
//TODO don't call here
@@ -116,23 +125,23 @@ func Detect(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.Sca
}
}
if c.Conf.DiffPlus || c.Conf.DiffMinus {
prevs, err := loadPrevious(rs)
if config.Conf.DiffPlus || config.Conf.DiffMinus {
prevs, err := loadPrevious(rs, config.Conf.ResultsDir)
if err != nil {
return nil, err
}
rs = diff(rs, prevs, c.Conf.DiffPlus, c.Conf.DiffMinus)
rs = diff(rs, prevs, config.Conf.DiffPlus, config.Conf.DiffMinus)
}
for i, r := range rs {
r.ScannedCves = r.ScannedCves.FilterByCvssOver(c.Conf.CvssScoreOver)
r.ScannedCves = r.ScannedCves.FilterUnfixed(c.Conf.IgnoreUnfixed)
r.ScannedCves = r.ScannedCves.FilterByCvssOver(config.Conf.CvssScoreOver)
r.ScannedCves = r.ScannedCves.FilterUnfixed(config.Conf.IgnoreUnfixed)
// IgnoreCves
ignoreCves := []string{}
if r.Container.Name == "" {
ignoreCves = c.Conf.Servers[r.ServerName].IgnoreCves
} else if con, ok := c.Conf.Servers[r.ServerName].Containers[r.Container.Name]; ok {
ignoreCves = config.Conf.Servers[r.ServerName].IgnoreCves
} else if con, ok := config.Conf.Servers[r.ServerName].Containers[r.Container.Name]; ok {
ignoreCves = con.IgnoreCves
}
r.ScannedCves = r.ScannedCves.FilterIgnoreCves(ignoreCves)
@@ -140,34 +149,35 @@ func Detect(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.Sca
// ignorePkgs
ignorePkgsRegexps := []string{}
if r.Container.Name == "" {
ignorePkgsRegexps = c.Conf.Servers[r.ServerName].IgnorePkgsRegexp
} else if s, ok := c.Conf.Servers[r.ServerName].Containers[r.Container.Name]; ok {
ignorePkgsRegexps = config.Conf.Servers[r.ServerName].IgnorePkgsRegexp
} else if s, ok := config.Conf.Servers[r.ServerName].Containers[r.Container.Name]; ok {
ignorePkgsRegexps = s.IgnorePkgsRegexp
}
r.ScannedCves = r.ScannedCves.FilterIgnorePkgs(ignorePkgsRegexps)
// IgnoreUnscored
if c.Conf.IgnoreUnscoredCves {
if config.Conf.IgnoreUnscoredCves {
r.ScannedCves = r.ScannedCves.FindScoredVulns()
}
r.FilterInactiveWordPressLibs(c.Conf.WpScan.DetectInactive)
r.FilterInactiveWordPressLibs(config.Conf.WpScan.DetectInactive)
rs[i] = r
}
return rs, nil
}
// DetectPkgCves detects OS pkg cves
func DetectPkgCves(dbclient DBClient, r *models.ScanResult) error {
// pass 2 configs
func DetectPkgCves(r *models.ScanResult, ovalCnf config.GovalDictConf, gostCnf config.GostConf) error {
// Pkg Scan
if r.Release != "" {
// OVAL
if err := detectPkgsCvesWithOval(dbclient.OvalDB, r); err != nil {
if err := detectPkgsCvesWithOval(ovalCnf, r); err != nil {
return xerrors.Errorf("Failed to detect CVE with OVAL: %w", err)
}
// gost
if err := detectPkgsCvesWithGost(dbclient.GostDB, r); err != nil {
if err := detectPkgsCvesWithGost(gostCnf, r); err != nil {
return xerrors.Errorf("Failed to detect CVE with gost: %w", err)
}
} else if reuseScannedCves(r) {
@@ -209,7 +219,7 @@ func DetectPkgCves(dbclient DBClient, r *models.ScanResult) error {
}
// DetectGitHubCves fetches CVEs from GitHub Security Alerts
func DetectGitHubCves(r *models.ScanResult, githubConfs map[string]c.GitHubConf, ignoreDismissed bool) error {
func DetectGitHubCves(r *models.ScanResult, githubConfs map[string]config.GitHubConf, ignoreDismissed bool) error {
if len(githubConfs) == 0 {
return nil
}
@@ -230,7 +240,7 @@ func DetectGitHubCves(r *models.ScanResult, githubConfs map[string]c.GitHubConf,
}
// DetectWordPressCves detects CVEs of WordPress
func DetectWordPressCves(r *models.ScanResult, wpCnf *c.WpScanConf) error {
func DetectWordPressCves(r *models.ScanResult, wpCnf config.WpScanConf) error {
if len(r.WordPressPackages) == 0 {
return nil
}
@@ -243,51 +253,33 @@ func DetectWordPressCves(r *models.ScanResult, wpCnf *c.WpScanConf) error {
return nil
}
// FillCveInfo fill scanResult with cve info.
func FillCveInfo(dbclient DBClient, r *models.ScanResult) error {
logging.Log.Infof("Fill CVE detailed with gost")
if err := gost.NewClient(r.Family).FillCVEsWithRedHat(dbclient.GostDB, r); err != nil {
return xerrors.Errorf("Failed to fill with gost: %w", err)
}
logging.Log.Infof("Fill CVE detailed with CVE-DB")
if err := fillCvesWithNvdJvn(dbclient.CveDB, r); err != nil {
return xerrors.Errorf("Failed to fill with CVE: %w", err)
}
logging.Log.Infof("Fill exploit with Exploit-DB")
nExploitCve, err := fillWithExploitDB(dbclient.ExploitDB, r)
if err != nil {
return xerrors.Errorf("Failed to fill with exploit: %w", err)
}
logging.Log.Infof("%s: %d exploits are detected",
r.FormatServerName(), nExploitCve)
logging.Log.Infof("Fill metasploit module with Metasploit-DB")
nMetasploitCve, err := fillWithMetasploit(dbclient.MetasploitDB, r)
if err != nil {
return xerrors.Errorf("Failed to fill with metasploit: %w", err)
}
logging.Log.Infof("%s: %d modules are detected",
r.FormatServerName(), nMetasploitCve)
logging.Log.Infof("Fill CWE with NVD")
fillCweDict(r)
return nil
}
// fillCvesWithNvdJvn fills CVE detail with NVD, JVN
func fillCvesWithNvdJvn(driver cvedb.DB, r *models.ScanResult) error {
// FillCvesWithNvdJvn fills CVE detail with NVD, JVN
func FillCvesWithNvdJvn(r *models.ScanResult, cnf config.GoCveDictConf, logOpts logging.LogOpts) (err error) {
cveIDs := []string{}
for _, v := range r.ScannedCves {
cveIDs = append(cveIDs, v.CveID)
}
ds, err := CveClient.FetchCveDetails(driver, cveIDs)
client, err := newGoCveDictClient(&cnf, logOpts)
if err != nil {
return err
}
defer func() {
if err := client.closeDB(); err != nil {
logging.Log.Errorf("Failed to close DB. err: %+v")
}
}()
var ds []cvemodels.CveDetail
if cnf.IsFetchViaHTTP() {
ds, err = client.fetchCveDetailsViaHTTP(cveIDs)
} else {
ds, err = client.fetchCveDetails(cveIDs)
}
if err != nil {
return err
}
for _, d := range ds {
nvd, exploits, mitigations := models.ConvertNvdJSONToModel(d.CveID, d.NvdJSON)
jvn := models.ConvertJvnToModel(d.CveID, d.Jvn)
@@ -337,72 +329,29 @@ func fillCertAlerts(cvedetail *cvemodels.CveDetail) (dict models.AlertDict) {
}
// detectPkgsCvesWithOval fetches OVAL database
func detectPkgsCvesWithOval(driver ovaldb.DB, r *models.ScanResult) error {
var ovalClient oval.Client
var ovalFamily string
switch r.Family {
case constant.Debian, constant.Raspbian:
ovalClient = oval.NewDebian()
ovalFamily = constant.Debian
case constant.Ubuntu:
ovalClient = oval.NewUbuntu()
ovalFamily = constant.Ubuntu
case constant.RedHat:
ovalClient = oval.NewRedhat()
ovalFamily = constant.RedHat
case constant.CentOS:
ovalClient = oval.NewCentOS()
//use RedHat's OVAL
ovalFamily = constant.RedHat
case constant.Oracle:
ovalClient = oval.NewOracle()
ovalFamily = constant.Oracle
case constant.SUSEEnterpriseServer:
// TODO other suse family
ovalClient = oval.NewSUSE()
ovalFamily = constant.SUSEEnterpriseServer
case constant.Alpine:
ovalClient = oval.NewAlpine()
ovalFamily = constant.Alpine
case constant.Amazon:
ovalClient = oval.NewAmazon()
ovalFamily = constant.Amazon
case constant.FreeBSD, constant.Windows:
return nil
case constant.ServerTypePseudo:
return nil
default:
if r.Family == "" {
return xerrors.New("Probably an error occurred during scanning. Check the error message")
}
return xerrors.Errorf("OVAL for %s is not implemented yet", r.Family)
func detectPkgsCvesWithOval(cnf config.GovalDictConf, r *models.ScanResult) error {
ovalClient, err := oval.NewOVALClient(r.Family, cnf)
if err != nil {
return err
}
if !c.Conf.OvalDict.IsFetchViaHTTP() {
if driver == nil {
return xerrors.Errorf("You have to fetch OVAL data for %s before reporting. For details, see `https://github.com/kotakanbe/goval-dictionary#usage`", r.Family)
}
if err := driver.NewOvalDB(ovalFamily); err != nil {
return xerrors.Errorf("Failed to New Oval DB. err: %w", err)
}
}
logging.Log.Debugf("Check whether oval fetched: %s %s", ovalFamily, r.Release)
ok, err := ovalClient.CheckIfOvalFetched(driver, ovalFamily, r.Release)
logging.Log.Debugf("Check if oval fetched: %s %s", r.Family, r.Release)
ok, err := ovalClient.CheckIfOvalFetched(r.Family, r.Release)
if err != nil {
return err
}
if !ok {
return xerrors.Errorf("OVAL entries of %s %s are not found. Fetch OVAL before reporting. For details, see `https://github.com/kotakanbe/goval-dictionary#usage`", ovalFamily, r.Release)
return xerrors.Errorf("OVAL entries of %s %s are not found. Fetch OVAL before reporting. For details, see `https://github.com/kotakanbe/goval-dictionary#usage`", r.Family, r.Release)
}
_, err = ovalClient.CheckIfOvalFresh(driver, ovalFamily, r.Release)
logging.Log.Debugf("Check if oval fresh: %s %s", r.Family, r.Release)
_, err = ovalClient.CheckIfOvalFresh(r.Family, r.Release)
if err != nil {
return err
}
nCVEs, err := ovalClient.FillWithOval(driver, r)
logging.Log.Debugf("Fill with oval: %s %s", r.Family, r.Release)
nCVEs, err := ovalClient.FillWithOval(r)
if err != nil {
return err
}
@@ -411,36 +360,36 @@ func detectPkgsCvesWithOval(driver ovaldb.DB, r *models.ScanResult) error {
return nil
}
func detectPkgsCvesWithGost(driver gostdb.DB, r *models.ScanResult) error {
nCVEs, err := gost.NewClient(r.Family).DetectUnfixed(driver, r, true)
func detectPkgsCvesWithGost(cnf config.GostConf, r *models.ScanResult) error {
client, err := gost.NewClient(cnf, r.Family)
if err != nil {
return xerrors.Errorf("Failed to new a gost client: %w", err)
}
logging.Log.Infof("%s: %d unfixed CVEs are detected with gost",
r.FormatServerName(), nCVEs)
return err
}
nCVEs, err := client.DetectUnfixed(r, true)
if err != nil {
return xerrors.Errorf("Failed to detect unfixed CVEs with gost: %w", err)
}
// fillWithExploitDB fills Exploits with exploit dataabase
// https://github.com/vulsio/go-exploitdb
func fillWithExploitDB(driver exploitdb.DB, r *models.ScanResult) (nExploitCve int, err error) {
return exploit.FillWithExploit(driver, r, &c.Conf.Exploit)
}
// fillWithMetasploit fills metasploit modules with metasploit database
// https://github.com/takuzoo3868/go-msfdb
func fillWithMetasploit(driver metasploitdb.DB, r *models.ScanResult) (nMetasploitCve int, err error) {
return msf.FillWithMetasploit(driver, r)
logging.Log.Infof("%s: %d unfixed CVEs are detected with gost", r.FormatServerName(), nCVEs)
return nil
}
// DetectCpeURIsCves detects CVEs of given CPE-URIs
func DetectCpeURIsCves(driver cvedb.DB, r *models.ScanResult, cpeURIs []string) error {
nCVEs := 0
if len(cpeURIs) != 0 && driver == nil && !c.Conf.CveDict.IsFetchViaHTTP() {
return xerrors.Errorf("cpeURIs %s specified, but cve-dictionary DB not found. Fetch cve-dictionary before reporting. For details, see `https://github.com/kotakanbe/go-cve-dictionary#deploy-go-cve-dictionary`",
cpeURIs)
func DetectCpeURIsCves(r *models.ScanResult, cpeURIs []string, cnf config.GoCveDictConf, logOpts logging.LogOpts) error {
client, err := newGoCveDictClient(&cnf, logOpts)
if err != nil {
return err
}
defer func() {
if err := client.closeDB(); err != nil {
logging.Log.Errorf("Failed to close DB. err: %+v")
}
}()
nCVEs := 0
for _, name := range cpeURIs {
details, err := CveClient.FetchCveDetailsByCpeName(driver, name)
details, err := client.fetchCveDetailsByCpeName(name)
if err != nil {
return err
}
@@ -466,7 +415,8 @@ func DetectCpeURIsCves(driver cvedb.DB, r *models.ScanResult, cpeURIs []string)
return nil
}
func fillCweDict(r *models.ScanResult) {
// FillCweDict fills CWE
func FillCweDict(r *models.ScanResult) {
uniqCweIDMap := map[string]bool{}
for _, vinfo := range r.ScannedCves {
for _, cont := range vinfo.CveContents {

223
detector/exploitdb.go Normal file
View File

@@ -0,0 +1,223 @@
// +build !scanner
package detector
import (
"encoding/json"
"net/http"
"time"
"github.com/cenkalti/backoff"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/logging"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
"github.com/parnurzeal/gorequest"
exploitdb "github.com/vulsio/go-exploitdb/db"
exploitmodels "github.com/vulsio/go-exploitdb/models"
"golang.org/x/xerrors"
)
// FillWithExploit fills exploit information that has in Exploit
func FillWithExploit(r *models.ScanResult, cnf config.ExploitConf) (nExploitCve int, err error) {
if cnf.IsFetchViaHTTP() {
var cveIDs []string
for cveID := range r.ScannedCves {
cveIDs = append(cveIDs, cveID)
}
prefix, _ := util.URLPathJoin(cnf.GetURL(), "cves")
responses, err := getCvesViaHTTP(cveIDs, prefix)
if err != nil {
return 0, err
}
for _, res := range responses {
exps := []*exploitmodels.Exploit{}
if err := json.Unmarshal([]byte(res.json), &exps); err != nil {
return 0, err
}
exploits := ConvertToModels(exps)
v, ok := r.ScannedCves[res.request.cveID]
if ok {
v.Exploits = exploits
}
r.ScannedCves[res.request.cveID] = v
nExploitCve++
}
} else {
driver, locked, err := newExploitDB(&cnf)
if locked {
return 0, xerrors.Errorf("SQLite3 is locked: %s", cnf.GetSQLite3Path())
} else if err != nil {
return 0, err
}
defer func() {
if err := driver.CloseDB(); err != nil {
logging.Log.Errorf("Failed to close DB. err: %+v")
}
}()
for cveID, vuln := range r.ScannedCves {
if cveID == "" {
continue
}
es := driver.GetExploitByCveID(cveID)
if len(es) == 0 {
continue
}
exploits := ConvertToModels(es)
vuln.Exploits = exploits
r.ScannedCves[cveID] = vuln
nExploitCve++
}
}
return nExploitCve, nil
}
// ConvertToModels converts gost model to vuls model
func ConvertToModels(es []*exploitmodels.Exploit) (exploits []models.Exploit) {
for _, e := range es {
var documentURL, shellURL *string
if e.OffensiveSecurity != nil {
os := e.OffensiveSecurity
if os.Document != nil {
documentURL = &os.Document.DocumentURL
}
if os.ShellCode != nil {
shellURL = &os.ShellCode.ShellCodeURL
}
}
exploit := models.Exploit{
ExploitType: e.ExploitType,
ID: e.ExploitUniqueID,
URL: e.URL,
Description: e.Description,
DocumentURL: documentURL,
ShellCodeURL: shellURL,
}
exploits = append(exploits, exploit)
}
return exploits
}
type exploitResponse struct {
request request
json string
}
func getCvesViaHTTP(cveIDs []string, urlPrefix string) (
responses []exploitResponse, err error) {
nReq := len(cveIDs)
reqChan := make(chan request, nReq)
resChan := make(chan exploitResponse, nReq)
errChan := make(chan error, nReq)
defer close(reqChan)
defer close(resChan)
defer close(errChan)
go func() {
for _, cveID := range cveIDs {
reqChan <- request{
cveID: cveID,
}
}
}()
concurrency := 10
tasks := util.GenWorkers(concurrency)
for i := 0; i < nReq; i++ {
tasks <- func() {
select {
case req := <-reqChan:
url, err := util.URLPathJoin(
urlPrefix,
req.cveID,
)
if err != nil {
errChan <- err
} else {
logging.Log.Debugf("HTTP Request to %s", url)
httpGet(url, req, resChan, errChan)
}
}
}
}
timeout := time.After(2 * 60 * time.Second)
var errs []error
for i := 0; i < nReq; i++ {
select {
case res := <-resChan:
responses = append(responses, res)
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
return nil, xerrors.New("Timeout Fetching OVAL")
}
}
if len(errs) != 0 {
return nil, xerrors.Errorf("Failed to fetch OVAL. err: %w", errs)
}
return
}
type request struct {
osMajorVersion string
packName string
isSrcPack bool
cveID string
}
func httpGet(url string, req request, resChan chan<- exploitResponse, errChan chan<- error) {
var body string
var errs []error
var resp *http.Response
count, retryMax := 0, 3
f := func() (err error) {
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
resp, body, errs = gorequest.New().Timeout(10 * time.Second).Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
count++
if count == retryMax {
return nil
}
return xerrors.Errorf("HTTP GET error, url: %s, resp: %v, err: %+v", url, resp, errs)
}
return nil
}
notify := func(err error, t time.Duration) {
logging.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
}
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
if err != nil {
errChan <- xerrors.Errorf("HTTP Error %w", err)
return
}
if count == retryMax {
errChan <- xerrors.New("Retry count exceeded")
return
}
resChan <- exploitResponse{
request: req,
json: body,
}
}
func newExploitDB(cnf config.VulnDictInterface) (driver exploitdb.DB, locked bool, err error) {
if cnf.IsFetchViaHTTP() {
return nil, false, nil
}
path := cnf.GetURL()
if cnf.GetType() == "sqlite3" {
path = cnf.GetSQLite3Path()
}
if driver, locked, err = exploitdb.NewDB(cnf.GetType(), path, cnf.GetDebugSQL()); err != nil {
if locked {
return nil, true, xerrors.Errorf("exploitDB is locked. err: %w", err)
}
return nil, false, err
}
return driver, false, nil
}

81
detector/msf.go Normal file
View File

@@ -0,0 +1,81 @@
// +build !scanner
package detector
import (
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/logging"
"github.com/future-architect/vuls/models"
metasploitdb "github.com/takuzoo3868/go-msfdb/db"
metasploitmodels "github.com/takuzoo3868/go-msfdb/models"
"golang.org/x/xerrors"
)
// FillWithMetasploit fills metasploit module information that has in module
func FillWithMetasploit(r *models.ScanResult, cnf config.MetasploitConf) (nMetasploitCve int, err error) {
driver, locked, err := newMetasploitDB(&cnf)
if locked {
return 0, xerrors.Errorf("SQLite3 is locked: %s", cnf.GetSQLite3Path())
} else if err != nil {
return 0, err
}
defer func() {
if err := driver.CloseDB(); err != nil {
logging.Log.Errorf("Failed to close DB. err: %+v")
}
}()
for cveID, vuln := range r.ScannedCves {
if cveID == "" {
continue
}
ms := driver.GetModuleByCveID(cveID)
if len(ms) == 0 {
continue
}
modules := ConvertToModelsMsf(ms)
vuln.Metasploits = modules
r.ScannedCves[cveID] = vuln
nMetasploitCve++
}
return nMetasploitCve, nil
}
// ConvertToModelsMsf converts gost model to vuls model
func ConvertToModelsMsf(ms []*metasploitmodels.Metasploit) (modules []models.Metasploit) {
for _, m := range ms {
var links []string
if 0 < len(m.References) {
for _, u := range m.References {
links = append(links, u.Link)
}
}
module := models.Metasploit{
Name: m.Name,
Title: m.Title,
Description: m.Description,
URLs: links,
}
modules = append(modules, module)
}
return modules
}
func newMetasploitDB(cnf config.VulnDictInterface) (driver metasploitdb.DB, locked bool, err error) {
if cnf.IsFetchViaHTTP() {
return nil, false, nil
}
path := cnf.GetURL()
if cnf.GetType() == "sqlite3" {
path = cnf.GetSQLite3Path()
}
if driver, locked, err = metasploitdb.NewDB(cnf.GetType(), path, cnf.GetDebugSQL(), false); err != nil {
if locked {
return nil, true, xerrors.Errorf("metasploitDB is locked. err: %w", err)
}
return nil, false, err
}
return driver, false, nil
}

View File

@@ -42,8 +42,8 @@ func needToRefreshCve(r models.ScanResult) bool {
return true
}
func loadPrevious(currs models.ScanResults) (prevs models.ScanResults, err error) {
dirs, err := ListValidJSONDirs()
func loadPrevious(currs models.ScanResults, resultsDir string) (prevs models.ScanResults, err error) {
dirs, err := ListValidJSONDirs(resultsDir)
if err != nil {
return
}
@@ -234,16 +234,16 @@ var jsonDirPattern = regexp.MustCompile(
// ListValidJSONDirs returns valid json directory as array
// Returned array is sorted so that recent directories are at the head
func ListValidJSONDirs() (dirs []string, err error) {
func ListValidJSONDirs(resultsDir string) (dirs []string, err error) {
var dirInfo []os.FileInfo
if dirInfo, err = ioutil.ReadDir(config.Conf.ResultsDir); err != nil {
if dirInfo, err = ioutil.ReadDir(resultsDir); err != nil {
err = xerrors.Errorf("Failed to read %s: %w",
config.Conf.ResultsDir, err)
return
}
for _, d := range dirInfo {
if d.IsDir() && jsonDirPattern.MatchString(d.Name()) {
jsonDir := filepath.Join(config.Conf.ResultsDir, d.Name())
jsonDir := filepath.Join(resultsDir, d.Name())
dirs = append(dirs, jsonDir)
}
}

View File

@@ -9,7 +9,7 @@ import (
"strings"
"time"
c "github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/errof"
"github.com/future-architect/vuls/logging"
"github.com/future-architect/vuls/models"
@@ -50,7 +50,7 @@ type References struct {
// DetectWordPressCves access to wpscan and fetch scurity alerts and then set to the given ScanResult.
// https://wpscan.com/
func detectWordPressCves(r *models.ScanResult, cnf *c.WpScanConf) (int, error) {
func detectWordPressCves(r *models.ScanResult, cnf config.WpScanConf) (int, error) {
if len(r.WordPressPackages) == 0 {
return 0, nil
}
@@ -230,7 +230,7 @@ func httpRequest(url, token string) (string, error) {
fmt.Sprintf("Failed to access to wpscan.com. err: %s", err))
}
req.Header.Set("Authorization", fmt.Sprintf("Token token=%s", token))
client, err := util.GetHTTPClient(c.Conf.HTTPProxy)
client, err := util.GetHTTPClient(config.Conf.HTTPProxy)
if err != nil {
return "", err
}