From e441e5a6960f6684496c831ae0fb88b70e374298 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Wed, 15 Mar 2017 16:44:49 +0900 Subject: [PATCH] Fix Bug of Mysql Backend --- README.ja.md | 6 +++--- README.md | 4 ++-- commands/report.go | 20 ++++++++++---------- commands/tui.go | 6 +++--- commands/util.go | 18 ++++++++---------- config/config.go | 30 +++++++++++++++++------------- cveapi/cve_client.go | 37 +++++++++++++++++++++---------------- models/models.go | 6 +++--- 8 files changed, 67 insertions(+), 60 deletions(-) diff --git a/README.ja.md b/README.ja.md index 46555f34..fcfc46f2 100644 --- a/README.ja.md +++ b/README.ja.md @@ -1240,9 +1240,9 @@ optional = [ ## Example: Use MySQL as a DB storage back-end ``` -$ vuls scan \ - -cve-dictionary-dbtype=mysql \ - -cve-dictionary-dbpath="user:pass@tcp(localhost:3306)/dbname?parseTime=true" +$ vuls report \ + -cvedb-type=mysql \ + -cvedb-url="user:pass@tcp(localhost:3306)/dbname" ``` ---- diff --git a/README.md b/README.md index a0887e44..237ce2b7 100644 --- a/README.md +++ b/README.md @@ -1232,9 +1232,9 @@ optional = [ ## Example: Use MySQL as a DB storage back-end ``` -$ vuls scan \ +$ vuls report \ -cvedb-type=mysql \ - -cvedb-url="user:pass@tcp(localhost:3306)/dbname?parseTime=true" + -cvedb-url="user:pass@tcp(localhost:3306)/dbname" ``` ---- diff --git a/commands/report.go b/commands/report.go index c8a548e0..b05b549e 100644 --- a/commands/report.go +++ b/commands/report.go @@ -46,9 +46,9 @@ type ReportCmd struct { ignoreUnscoredCves bool httpProxy string - cvedbtype string - cvedbpath string - cveDictionaryURL string + cvedbtype string + cvedbpath string + cvedbURL string toSlack bool toEMail bool @@ -160,7 +160,7 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) { "/path/to/sqlite3 (For get cve detail from cve.sqlite3)") f.StringVar( - &p.cveDictionaryURL, + &p.cvedbURL, "cvedb-url", "", "http://cve-dictionary.com:8080 or mysql connection string") @@ -267,7 +267,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} c.Conf.ResultsDir = p.resultsDir c.Conf.CveDBType = p.cvedbtype c.Conf.CveDBPath = p.cvedbpath - c.Conf.CveDictionaryURL = p.cveDictionaryURL + c.Conf.CveDBURL = p.cvedbURL c.Conf.CvssScoreOver = p.cvssScoreOver c.Conf.IgnoreUnscoredCves = p.ignoreUnscoredCves c.Conf.HTTPProxy = p.httpProxy @@ -355,8 +355,8 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} util.Log.Errorf("Run go-cve-dictionary as server mode before reporting or run with --cvedb-path option") return subcommands.ExitFailure } - if c.Conf.CveDictionaryURL != "" { - util.Log.Infof("cve-dictionary: %s", c.Conf.CveDictionaryURL) + if c.Conf.CveDBURL != "" { + util.Log.Infof("cve-dictionary: %s", c.Conf.CveDBURL) } else { if c.Conf.CveDBType == "sqlite3" { util.Log.Infof("cve-dictionary: %s", c.Conf.CveDBPath) @@ -374,7 +374,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} for _, r := range history.ScanResults { if p.refreshCve || needToRefreshCve(r) { util.Log.Debugf("need to refresh") - if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDictionaryURL == "" { + if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDBURL == "" { if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) { util.Log.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s", c.Conf.CveDBPath) @@ -389,11 +389,11 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } filled.Lang = c.Conf.Lang - if err := overwriteJSONFile(jsonDir, filled); err != nil { + if err := overwriteJSONFile(jsonDir, *filled); err != nil { util.Log.Errorf("Failed to write JSON: %s", err) return subcommands.ExitFailure } - results = append(results, filled) + results = append(results, *filled) } else { util.Log.Debugf("no need to refresh") results = append(results, r) diff --git a/commands/tui.go b/commands/tui.go index 25aa5bd0..5e9fa774 100644 --- a/commands/tui.go +++ b/commands/tui.go @@ -113,7 +113,7 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s c.Conf.ResultsDir = p.resultsDir c.Conf.CveDBType = p.cvedbtype c.Conf.CveDBPath = p.cvedbpath - c.Conf.CveDictionaryURL = p.cveDictionaryURL + c.Conf.CveDBURL = p.cveDictionaryURL log.Info("Validating config...") if !c.Conf.ValidateOnTui() { @@ -150,11 +150,11 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s return subcommands.ExitFailure } - if err := overwriteJSONFile(jsonDir, filled); err != nil { + if err := overwriteJSONFile(jsonDir, *filled); err != nil { log.Errorf("Failed to write JSON: %s", err) return subcommands.ExitFailure } - results = append(results, filled) + results = append(results, *filled) } else { results = append(results, r) } diff --git a/commands/util.go b/commands/util.go index 11bdb6d8..f11473f1 100644 --- a/commands/util.go +++ b/commands/util.go @@ -157,18 +157,17 @@ func loadOneScanHistory(jsonDir string) (scanHistory models.ScanHistory, err err return } -func fillCveInfoFromCveDB(r models.ScanResult) (filled models.ScanResult, err error) { +func fillCveInfoFromCveDB(r models.ScanResult) (*models.ScanResult, error) { + var err error + var vs []models.VulnInfo + sInfo := c.Conf.Servers[r.ServerName] - vs, err := scanVulnByCpeNames(sInfo.CpeNames, r.ScannedCves) + vs, err = scanVulnByCpeNames(sInfo.CpeNames, r.ScannedCves) if err != nil { - return + return nil, err } r.ScannedCves = vs - filled, err = r.FillCveDetail() - if err != nil { - return - } - return + return r.FillCveDetail() } func overwriteJSONFile(dir string, r models.ScanResult) error { @@ -182,8 +181,7 @@ func overwriteJSONFile(dir string, r models.ScanResult) error { return nil } -func scanVulnByCpeNames(cpeNames []string, scannedVulns []models.VulnInfo) ([]models.VulnInfo, - error) { +func scanVulnByCpeNames(cpeNames []string, scannedVulns []models.VulnInfo) ([]models.VulnInfo, error) { // To remove duplicate set := map[string]models.VulnInfo{} for _, v := range scannedVulns { diff --git a/config/config.go b/config/config.go index e7cf4eee..eb843149 100644 --- a/config/config.go +++ b/config/config.go @@ -41,21 +41,20 @@ type Config struct { Default ServerInfo Servers map[string]ServerInfo - CveDictionaryURL string `valid:"url"` - CvssScoreOver float64 IgnoreUnscoredCves bool - AssumeYes bool SSHExternal bool ContainersOnly bool SkipBroken bool - HTTPProxy string `valid:"url"` - LogDir string - ResultsDir string + HTTPProxy string `valid:"url"` + LogDir string + ResultsDir string + CveDBType string CveDBPath string + CveDBURL string CacheDBPath string FormatXML bool @@ -155,16 +154,21 @@ func (c Config) ValidateOnReport() bool { } } - if c.CveDBType != "sqlite3" && c.CveDBType != "mysql" { - errs = append(errs, fmt.Errorf( - "CVE DB type must be either 'sqlite3' or 'mysql'. -cve-dictionary-dbtype: %s", c.CveDBType)) - } - - if c.CveDBType == "sqlite3" { + switch c.CveDBType { + case "sqlite3": if ok, _ := valid.IsFilePath(c.CveDBPath); !ok { errs = append(errs, fmt.Errorf( - "SQLite3 DB(CVE-Dictionary) path must be a *Absolute* file path. -cve-dictionary-dbpath: %s", c.CveDBPath)) + "SQLite3 DB(CVE-Dictionary) path must be a *Absolute* file path. -cvedb-path: %s", + c.CveDBPath)) } + case "mysql": + if c.CveDBURL == "" { + errs = append(errs, fmt.Errorf( + `MySQL connection string is needed. -cvedb-url="user:pass@tcp(localhost:3306)/dbname"`)) + } + default: + errs = append(errs, fmt.Errorf( + "CVE DB type must be either 'sqlite3' or 'mysql'. -cvedb-type: %s", c.CveDBType)) } _, err := valid.ValidateStruct(c) diff --git a/cveapi/cve_client.go b/cveapi/cve_client.go index 4560c43a..4adfa897 100644 --- a/cveapi/cve_client.go +++ b/cveapi/cve_client.go @@ -27,7 +27,6 @@ import ( "github.com/cenkalti/backoff" "github.com/parnurzeal/gorequest" - log "github.com/Sirupsen/logrus" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/util" cveconfig "github.com/kotakanbe/go-cve-dictionary/config" @@ -44,12 +43,12 @@ type cvedictClient struct { } func (api *cvedictClient) initialize() { - api.baseURL = config.Conf.CveDictionaryURL + api.baseURL = config.Conf.CveDBURL } func (api cvedictClient) CheckHealth() (ok bool, err error) { - if config.Conf.CveDictionaryURL == "" { - log.Debugf("get cve-dictionary from %s", config.Conf.CveDBType) + if config.Conf.CveDBURL == "" || config.Conf.CveDBType == "mysql" { + util.Log.Debugf("get cve-dictionary from %s", config.Conf.CveDBType) return true, nil } @@ -71,11 +70,12 @@ type response struct { } func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDetails, err error) { - if config.Conf.CveDictionaryURL == "" { + switch config.Conf.CveDBType { + case "sqlite3", "mysql": return api.FetchCveDetailsFromCveDB(cveIDs) } - api.baseURL = config.Conf.CveDictionaryURL + api.baseURL = config.Conf.CveDBURL reqChan := make(chan string, len(cveIDs)) resChan := make(chan response, len(cveIDs)) errChan := make(chan error, len(cveIDs)) @@ -99,7 +99,7 @@ func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDet if err != nil { errChan <- err } else { - log.Debugf("HTTP Request to %s", url) + util.Log.Debugf("HTTP Request to %s", url) api.httpGet(cveID, url, resChan, errChan) } } @@ -134,12 +134,12 @@ func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDet } func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails cve.CveDetails, err error) { - log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType) + util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType) cveconfig.Conf.DBType = config.Conf.CveDBType if config.Conf.CveDBType == "sqlite3" { cveconfig.Conf.DBPath = config.Conf.CveDBPath } else { - cveconfig.Conf.DBPath = config.Conf.CveDictionaryURL + cveconfig.Conf.DBPath = config.Conf.CveDBURL } cveconfig.Conf.DebugSQL = config.Conf.DebugSQL if err := cvedb.OpenDB(); err != nil { @@ -175,7 +175,7 @@ func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errCh return nil } notify := func(err error, t time.Duration) { - log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err) + util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err) } err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) if err != nil { @@ -197,18 +197,19 @@ type responseGetCveDetailByCpeName struct { } func (api cvedictClient) FetchCveDetailsByCpeName(cpeName string) ([]cve.CveDetail, error) { - if config.Conf.CveDictionaryURL == "" { + switch config.Conf.CveDBType { + case "sqlite3", "mysql": return api.FetchCveDetailsByCpeNameFromDB(cpeName) } - api.baseURL = config.Conf.CveDictionaryURL + api.baseURL = config.Conf.CveDBURL url, err := util.URLPathJoin(api.baseURL, "cpes") if err != nil { return []cve.CveDetail{}, err } query := map[string]string{"name": cpeName} - log.Debugf("HTTP Request to %s, query: %#v", url, query) + util.Log.Debugf("HTTP Request to %s, query: %#v", url, query) return api.httpPost(cpeName, url, query) } @@ -228,7 +229,7 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c return nil } notify := func(err error, t time.Duration) { - log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %s", t, err) + util.Log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %s", t, err) } err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) if err != nil { @@ -244,9 +245,13 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c } func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) ([]cve.CveDetail, error) { - log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType) + util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType) cveconfig.Conf.DBType = config.Conf.CveDBType - cveconfig.Conf.DBPath = config.Conf.CveDBPath + if config.Conf.CveDBType == "sqlite3" { + cveconfig.Conf.DBPath = config.Conf.CveDBPath + } else { + cveconfig.Conf.DBPath = config.Conf.CveDBURL + } cveconfig.Conf.DebugSQL = config.Conf.DebugSQL if err := cvedb.OpenDB(); err != nil { diff --git a/models/models.go b/models/models.go index f13d10fd..ddc94a3d 100644 --- a/models/models.go +++ b/models/models.go @@ -79,7 +79,7 @@ type ScanResult struct { // FillCveDetail fetches CVE detailed information from // CVE Database, and then set to fields. -func (r ScanResult) FillCveDetail() (ScanResult, error) { +func (r ScanResult) FillCveDetail() (*ScanResult, error) { set := map[string]VulnInfo{} var cveIDs []string for _, v := range r.ScannedCves { @@ -89,7 +89,7 @@ func (r ScanResult) FillCveDetail() (ScanResult, error) { ds, err := cveapi.CveClient.FetchCveDetails(cveIDs) if err != nil { - return r, err + return nil, err } known, unknown, ignored := CveInfos{}, CveInfos{}, CveInfos{} @@ -128,7 +128,7 @@ func (r ScanResult) FillCveDetail() (ScanResult, error) { r.KnownCves = known r.UnknownCves = unknown r.IgnoredCves = ignored - return r, nil + return &r, nil } // FilterByCvssOver is filter function.