Merge pull request #384 from future-architect/mysql

Fix Bug of Mysql Backend

README.ja.md

@@ -1240,9 +1240,9 @@ optional = [
 ## Example: Use MySQL as a DB storage back-end
 
 ```
-$ vuls scan \
-      -cve-dictionary-dbtype=mysql \
-      -cve-dictionary-dbpath="user:pass@tcp(localhost:3306)/dbname?parseTime=true"
+$ vuls report \
+      -cvedb-type=mysql \
+      -cvedb-url="user:pass@tcp(localhost:3306)/dbname"
 ```
 
 ----

README.md

@@ -1232,9 +1232,9 @@ optional = [
 ## Example: Use MySQL as a DB storage back-end
 
 ```
-$ vuls scan \
+$ vuls report \
       -cvedb-type=mysql \
-      -cvedb-url="user:pass@tcp(localhost:3306)/dbname?parseTime=true"
+      -cvedb-url="user:pass@tcp(localhost:3306)/dbname"
 ```
 
 ----

commands/report.go

@@ -46,9 +46,9 @@ type ReportCmd struct {
 	ignoreUnscoredCves bool
 	httpProxy          string
 
-	cvedbtype        string
-	cvedbpath        string
-	cveDictionaryURL string
+	cvedbtype string
+	cvedbpath string
+	cvedbURL  string
 
 	toSlack     bool
 	toEMail     bool
@@ -160,7 +160,7 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) {
 		"/path/to/sqlite3 (For get cve detail from cve.sqlite3)")
 
 	f.StringVar(
-		&p.cveDictionaryURL,
+		&p.cvedbURL,
 		"cvedb-url",
 		"",
 		"http://cve-dictionary.com:8080 or mysql connection string")
@@ -267,7 +267,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 	c.Conf.ResultsDir = p.resultsDir
 	c.Conf.CveDBType = p.cvedbtype
 	c.Conf.CveDBPath = p.cvedbpath
-	c.Conf.CveDictionaryURL = p.cveDictionaryURL
+	c.Conf.CveDBURL = p.cvedbURL
 	c.Conf.CvssScoreOver = p.cvssScoreOver
 	c.Conf.IgnoreUnscoredCves = p.ignoreUnscoredCves
 	c.Conf.HTTPProxy = p.httpProxy
@@ -355,8 +355,8 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 		util.Log.Errorf("Run go-cve-dictionary as server mode before reporting or run with --cvedb-path option")
 		return subcommands.ExitFailure
 	}
-	if c.Conf.CveDictionaryURL != "" {
-		util.Log.Infof("cve-dictionary: %s", c.Conf.CveDictionaryURL)
+	if c.Conf.CveDBURL != "" {
+		util.Log.Infof("cve-dictionary: %s", c.Conf.CveDBURL)
 	} else {
 		if c.Conf.CveDBType == "sqlite3" {
 			util.Log.Infof("cve-dictionary: %s", c.Conf.CveDBPath)
@@ -374,7 +374,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 	for _, r := range history.ScanResults {
 		if p.refreshCve || needToRefreshCve(r) {
 			util.Log.Debugf("need to refresh")
-			if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDictionaryURL == "" {
+			if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDBURL == "" {
 				if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) {
 					util.Log.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s",
 						c.Conf.CveDBPath)
@@ -389,11 +389,11 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 			}
 			filled.Lang = c.Conf.Lang
 
-			if err := overwriteJSONFile(jsonDir, filled); err != nil {
+			if err := overwriteJSONFile(jsonDir, *filled); err != nil {
 				util.Log.Errorf("Failed to write JSON: %s", err)
 				return subcommands.ExitFailure
 			}
-			results = append(results, filled)
+			results = append(results, *filled)
 		} else {
 			util.Log.Debugf("no need to refresh")
 			results = append(results, r)

commands/tui.go

@@ -113,7 +113,7 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
 	c.Conf.ResultsDir = p.resultsDir
 	c.Conf.CveDBType = p.cvedbtype
 	c.Conf.CveDBPath = p.cvedbpath
-	c.Conf.CveDictionaryURL = p.cveDictionaryURL
+	c.Conf.CveDBURL = p.cveDictionaryURL
 
 	log.Info("Validating config...")
 	if !c.Conf.ValidateOnTui() {
@@ -150,11 +150,11 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
 				return subcommands.ExitFailure
 			}
 
-			if err := overwriteJSONFile(jsonDir, filled); err != nil {
+			if err := overwriteJSONFile(jsonDir, *filled); err != nil {
 				log.Errorf("Failed to write JSON: %s", err)
 				return subcommands.ExitFailure
 			}
-			results = append(results, filled)
+			results = append(results, *filled)
 		} else {
 			results = append(results, r)
 		}

commands/util.go

@@ -157,18 +157,17 @@ func loadOneScanHistory(jsonDir string) (scanHistory models.ScanHistory, err err
 	return
 }
 
-func fillCveInfoFromCveDB(r models.ScanResult) (filled models.ScanResult, err error) {
+func fillCveInfoFromCveDB(r models.ScanResult) (*models.ScanResult, error) {
+	var err error
+	var vs []models.VulnInfo
+
 	sInfo := c.Conf.Servers[r.ServerName]
-	vs, err := scanVulnByCpeNames(sInfo.CpeNames, r.ScannedCves)
+	vs, err = scanVulnByCpeNames(sInfo.CpeNames, r.ScannedCves)
 	if err != nil {
-		return
+		return nil, err
 	}
 	r.ScannedCves = vs
-	filled, err = r.FillCveDetail()
-	if err != nil {
-		return
-	}
-	return
+	return r.FillCveDetail()
 }
 
 func overwriteJSONFile(dir string, r models.ScanResult) error {
@@ -182,8 +181,7 @@ func overwriteJSONFile(dir string, r models.ScanResult) error {
 	return nil
 }
 
-func scanVulnByCpeNames(cpeNames []string, scannedVulns []models.VulnInfo) ([]models.VulnInfo,
-	error) {
+func scanVulnByCpeNames(cpeNames []string, scannedVulns []models.VulnInfo) ([]models.VulnInfo, error) {
 	// To remove duplicate
 	set := map[string]models.VulnInfo{}
 	for _, v := range scannedVulns {

config/config.go

@@ -41,21 +41,20 @@ type Config struct {
 	Default ServerInfo
 	Servers map[string]ServerInfo
 
-	CveDictionaryURL string `valid:"url"`
-
 	CvssScoreOver      float64
 	IgnoreUnscoredCves bool
 
-	AssumeYes      bool
 	SSHExternal    bool
 	ContainersOnly bool
 	SkipBroken     bool
 
-	HTTPProxy   string `valid:"url"`
-	LogDir      string
-	ResultsDir  string
+	HTTPProxy  string `valid:"url"`
+	LogDir     string
+	ResultsDir string
+
 	CveDBType   string
 	CveDBPath   string
+	CveDBURL    string
 	CacheDBPath string
 
 	FormatXML         bool
@@ -155,16 +154,21 @@ func (c Config) ValidateOnReport() bool {
 		}
 	}
 
-	if c.CveDBType != "sqlite3" && c.CveDBType != "mysql" {
-		errs = append(errs, fmt.Errorf(
-			"CVE DB type must be either 'sqlite3' or 'mysql'.  -cve-dictionary-dbtype: %s", c.CveDBType))
-	}
-
-	if c.CveDBType == "sqlite3" {
+	switch c.CveDBType {
+	case "sqlite3":
 		if ok, _ := valid.IsFilePath(c.CveDBPath); !ok {
 			errs = append(errs, fmt.Errorf(
-				"SQLite3 DB(CVE-Dictionary) path must be a *Absolute* file path. -cve-dictionary-dbpath: %s", c.CveDBPath))
+				"SQLite3 DB(CVE-Dictionary) path must be a *Absolute* file path. -cvedb-path: %s",
+				c.CveDBPath))
 		}
+	case "mysql":
+		if c.CveDBURL == "" {
+			errs = append(errs, fmt.Errorf(
+				`MySQL connection string is needed. -cvedb-url="user:pass@tcp(localhost:3306)/dbname"`))
+		}
+	default:
+		errs = append(errs, fmt.Errorf(
+			"CVE DB type must be either 'sqlite3' or 'mysql'.  -cvedb-type: %s", c.CveDBType))
 	}
 
 	_, err := valid.ValidateStruct(c)

cveapi/cve_client.go

@@ -27,7 +27,6 @@ import (
 	"github.com/cenkalti/backoff"
 	"github.com/parnurzeal/gorequest"
 
-	log "github.com/Sirupsen/logrus"
 	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/util"
 	cveconfig "github.com/kotakanbe/go-cve-dictionary/config"
@@ -44,12 +43,12 @@ type cvedictClient struct {
 }
 
 func (api *cvedictClient) initialize() {
-	api.baseURL = config.Conf.CveDictionaryURL
+	api.baseURL = config.Conf.CveDBURL
 }
 
 func (api cvedictClient) CheckHealth() (ok bool, err error) {
-	if config.Conf.CveDictionaryURL == "" {
-		log.Debugf("get cve-dictionary from %s", config.Conf.CveDBType)
+	if config.Conf.CveDBURL == "" || config.Conf.CveDBType == "mysql" {
+		util.Log.Debugf("get cve-dictionary from %s", config.Conf.CveDBType)
 		return true, nil
 	}
 
@@ -71,11 +70,12 @@ type response struct {
 }
 
 func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDetails, err error) {
-	if config.Conf.CveDictionaryURL == "" {
+	switch config.Conf.CveDBType {
+	case "sqlite3", "mysql":
 		return api.FetchCveDetailsFromCveDB(cveIDs)
 	}
 
-	api.baseURL = config.Conf.CveDictionaryURL
+	api.baseURL = config.Conf.CveDBURL
 	reqChan := make(chan string, len(cveIDs))
 	resChan := make(chan response, len(cveIDs))
 	errChan := make(chan error, len(cveIDs))
@@ -99,7 +99,7 @@ func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDet
 				if err != nil {
 					errChan <- err
 				} else {
-					log.Debugf("HTTP Request to %s", url)
+					util.Log.Debugf("HTTP Request to %s", url)
 					api.httpGet(cveID, url, resChan, errChan)
 				}
 			}
@@ -134,12 +134,12 @@ func (api cvedictClient) FetchCveDetails(cveIDs []string) (cveDetails cve.CveDet
 }
 
 func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails cve.CveDetails, err error) {
-	log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType)
+	util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType)
 	cveconfig.Conf.DBType = config.Conf.CveDBType
 	if config.Conf.CveDBType == "sqlite3" {
 		cveconfig.Conf.DBPath = config.Conf.CveDBPath
 	} else {
-		cveconfig.Conf.DBPath = config.Conf.CveDictionaryURL
+		cveconfig.Conf.DBPath = config.Conf.CveDBURL
 	}
 	cveconfig.Conf.DebugSQL = config.Conf.DebugSQL
 	if err := cvedb.OpenDB(); err != nil {
@@ -175,7 +175,7 @@ func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errCh
 		return nil
 	}
 	notify := func(err error, t time.Duration) {
-		log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
+		util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
 	}
 	err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
 	if err != nil {
@@ -197,18 +197,19 @@ type responseGetCveDetailByCpeName struct {
 }
 
 func (api cvedictClient) FetchCveDetailsByCpeName(cpeName string) ([]cve.CveDetail, error) {
-	if config.Conf.CveDictionaryURL == "" {
+	switch config.Conf.CveDBType {
+	case "sqlite3", "mysql":
 		return api.FetchCveDetailsByCpeNameFromDB(cpeName)
 	}
 
-	api.baseURL = config.Conf.CveDictionaryURL
+	api.baseURL = config.Conf.CveDBURL
 	url, err := util.URLPathJoin(api.baseURL, "cpes")
 	if err != nil {
 		return []cve.CveDetail{}, err
 	}
 
 	query := map[string]string{"name": cpeName}
-	log.Debugf("HTTP Request to %s, query: %#v", url, query)
+	util.Log.Debugf("HTTP Request to %s, query: %#v", url, query)
 	return api.httpPost(cpeName, url, query)
 }
 
@@ -228,7 +229,7 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c
 		return nil
 	}
 	notify := func(err error, t time.Duration) {
-		log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %s", t, err)
+		util.Log.Warnf("Failed to HTTP POST. retrying in %s seconds. err: %s", t, err)
 	}
 	err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
 	if err != nil {
@@ -244,9 +245,13 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c
 }
 
 func (api cvedictClient) FetchCveDetailsByCpeNameFromDB(cpeName string) ([]cve.CveDetail, error) {
-	log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType)
+	util.Log.Debugf("open cve-dictionary db (%s)", config.Conf.CveDBType)
 	cveconfig.Conf.DBType = config.Conf.CveDBType
-	cveconfig.Conf.DBPath = config.Conf.CveDBPath
+	if config.Conf.CveDBType == "sqlite3" {
+		cveconfig.Conf.DBPath = config.Conf.CveDBPath
+	} else {
+		cveconfig.Conf.DBPath = config.Conf.CveDBURL
+	}
 	cveconfig.Conf.DebugSQL = config.Conf.DebugSQL
 
 	if err := cvedb.OpenDB(); err != nil {

models/models.go

@@ -79,7 +79,7 @@ type ScanResult struct {
 
 // FillCveDetail fetches CVE detailed information from
 // CVE Database, and then set to fields.
-func (r ScanResult) FillCveDetail() (ScanResult, error) {
+func (r ScanResult) FillCveDetail() (*ScanResult, error) {
 	set := map[string]VulnInfo{}
 	var cveIDs []string
 	for _, v := range r.ScannedCves {
@@ -89,7 +89,7 @@ func (r ScanResult) FillCveDetail() (ScanResult, error) {
 
 	ds, err := cveapi.CveClient.FetchCveDetails(cveIDs)
 	if err != nil {
-		return r, err
+		return nil, err
 	}
 
 	known, unknown, ignored := CveInfos{}, CveInfos{}, CveInfos{}
@@ -128,7 +128,7 @@ func (r ScanResult) FillCveDetail() (ScanResult, error) {
 	r.KnownCves = known
 	r.UnknownCves = unknown
 	r.IgnoredCves = ignored
-	return r, nil
+	return &r, nil
 }
 
 // FilterByCvssOver is filter function.