Implement HTTP access to oval-dictionary

2017-06-05 17:37:02 +09:00
parent 997dd6022f
commit 1883da3b2a
8 changed files with 277 additions and 69 deletions
--- a/commands/report.go
+++ b/commands/report.go
@@ -45,12 +45,13 @@ type ReportCmd struct {
 	ignoreUnscoredCves bool
 	httpProxy          string

-	cvedbtype string
-	cvedbpath string
-	cvedbURL  string
+	cveDBType string
+	cveDBPath string
+	cveDBURL  string

-	ovaldbtype string
-	ovaldbpath string
+	ovalDBType string
+	ovalDBPath string
+	ovalDBURL  string

 	toSlack     bool
 	toEMail     bool
@@ -98,6 +99,9 @@ func (*ReportCmd) Usage() string {
 		[-cvedb-type=sqlite3|mysql|postgres]
 		[-cvedb-path=/path/to/cve.sqlite3]
 		[-cvedb-url=http://127.0.0.1:1323 or DB connection string]
+		[-ovaldb-type=sqlite3|mysql]
+		[-ovaldb-path=/path/to/oval.sqlite3]
+		[-ovaldb-url=http://127.0.0.1:1324 or DB connection string]
 		[-cvss-over=7]
 		[-diff]
 		[-ignore-unscored-cves]
@@ -152,36 +156,42 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) {
 		"Refresh CVE information in JSON file under results dir")

 	f.StringVar(
-		&p.cvedbtype,
+		&p.cveDBType,
 		"cvedb-type",
 		"sqlite3",
 		"DB type for fetching CVE dictionary (sqlite3, mysql or postgres)")

 	defaultCveDBPath := filepath.Join(wd, "cve.sqlite3")
 	f.StringVar(
-		&p.cvedbpath,
+		&p.cveDBPath,
 		"cvedb-path",
 		defaultCveDBPath,
 		"/path/to/sqlite3 (For get cve detail from cve.sqlite3)")

 	f.StringVar(
-		&p.ovaldbtype,
+		&p.cveDBURL,
+		"cvedb-url",
+		"",
+		"http://cve-dictionary.com:1323 or mysql connection string")
+
+	f.StringVar(
+		&p.ovalDBType,
 		"ovaldb-type",
 		"sqlite3",
 		"DB type for fetching OVAL dictionary (sqlite3 or mysql)")

 	defaultOvalDBPath := filepath.Join(wd, "oval.sqlite3")
 	f.StringVar(
-		&p.ovaldbpath,
+		&p.ovalDBPath,
 		"ovaldb-path",
 		defaultOvalDBPath,
 		"/path/to/sqlite3 (For get oval detail from oval.sqlite3)")

 	f.StringVar(
-		&p.cvedbURL,
-		"cvedb-url",
+		&p.ovalDBURL,
+		"ovaldb-url",
 		"",
-		"http://cve-dictionary.com:8080 or DB connection string")
+		"http://goval-dictionary.com:1324 or mysql connection string")

 	f.Float64Var(
 		&p.cvssScoreOver,
@@ -290,11 +300,12 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 	c.Conf.ResultsDir = p.resultsDir
 	c.Conf.RefreshCve = p.refreshCve
 	c.Conf.Diff = p.diff
-	c.Conf.CveDBType = p.cvedbtype
-	c.Conf.CveDBPath = p.cvedbpath
-	c.Conf.CveDBURL = p.cvedbURL
-	c.Conf.OvalDBType = p.ovaldbtype
-	c.Conf.OvalDBPath = p.ovaldbpath
+	c.Conf.CveDBType = p.cveDBType
+	c.Conf.CveDBPath = p.cveDBPath
+	c.Conf.CveDBURL = p.cveDBURL
+	c.Conf.OvalDBType = p.ovalDBType
+	c.Conf.OvalDBPath = p.ovalDBPath
+	c.Conf.OvalDBURL = p.ovalDBURL
 	c.Conf.CvssScoreOver = p.cvssScoreOver
 	c.Conf.IgnoreUnscoredCves = p.ignoreUnscoredCves
 	c.Conf.HTTPProxy = p.httpProxy
--- a/commands/tui.go
+++ b/commands/tui.go
@@ -38,12 +38,17 @@ type TuiCmd struct {
 	configPath string
 	logDir     string

-	resultsDir       string
-	refreshCve       bool
+	resultsDir string
+	refreshCve bool
+
 	cvedbtype        string
 	cvedbpath        string
 	cveDictionaryURL string

+	ovalDBType string
+	ovalDBPath string
+	ovalDBURL  string
+
 	pipe bool
 }

@@ -61,6 +66,9 @@ func (*TuiCmd) Usage() string {
 		[-cvedb-type=sqlite3|mysql|postgres]
 		[-cvedb-path=/path/to/cve.sqlite3]
 		[-cvedb-url=http://127.0.0.1:1323 or DB connection string]
+		[-ovaldb-type=sqlite3|mysql]
+		[-ovaldb-path=/path/to/oval.sqlite3]
+		[-ovaldb-url=http://127.0.0.1:1324 or DB connection string]
 		[-refresh-cve]
 		[-results-dir=/path/to/results]
 		[-log-dir=/path/to/log]
@@ -110,7 +118,26 @@ func (p *TuiCmd) SetFlags(f *flag.FlagSet) {
 		&p.cveDictionaryURL,
 		"cvedb-url",
 		"",
-		"http://cve-dictionary.com:8080 or DB connection string")
+		"http://cve-dictionary.com:1323 or mysql connection string")
+
+	f.StringVar(
+		&p.ovalDBType,
+		"ovaldb-type",
+		"sqlite3",
+		"DB type for fetching OVAL dictionary (sqlite3 or mysql)")
+
+	defaultOvalDBPath := filepath.Join(wd, "oval.sqlite3")
+	f.StringVar(
+		&p.ovalDBPath,
+		"ovaldb-path",
+		defaultOvalDBPath,
+		"/path/to/sqlite3 (For get oval detail from oval.sqlite3)")
+
+	f.StringVar(
+		&p.ovalDBURL,
+		"ovaldb-url",
+		"",
+		"http://goval-dictionary.com:1324 or mysql connection string")

 	f.BoolVar(
 		&p.pipe,
@@ -139,6 +166,9 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
 	c.Conf.CveDBType = p.cvedbtype
 	c.Conf.CveDBPath = p.cvedbpath
 	c.Conf.CveDBURL = p.cveDictionaryURL
+	c.Conf.OvalDBType = p.ovalDBType
+	c.Conf.OvalDBPath = p.ovalDBPath
+	c.Conf.OvalDBURL = p.ovalDBURL

 	log.Info("Validating config...")
 	if !c.Conf.ValidateOnTui() {
--- a/config/config.go
+++ b/config/config.go
@@ -53,13 +53,15 @@ type Config struct {
 	LogDir     string
 	ResultsDir string

-	CveDBType   string
-	CveDBPath   string
-	CveDBURL    string
-	CacheDBPath string
+	CveDBType string
+	CveDBPath string
+	CveDBURL  string

 	OvalDBType string
 	OvalDBPath string
+	OvalDBURL  string
+
+	CacheDBPath string

 	RefreshCve bool

--- a/oval/debian.go
+++ b/oval/debian.go
@@ -15,8 +15,8 @@ import (
 // DebianBase is the base struct of Debian and Ubuntu
 type DebianBase struct{}

-// fillCveInfoFromOvalDB returns scan result after updating CVE info by OVAL
-func (o DebianBase) fillCveInfoFromOvalDB(r *models.ScanResult) error {
+// fillFromOvalDB returns scan result after updating CVE info by OVAL
+func (o DebianBase) fillFromOvalDB(r *models.ScanResult) error {
 	ovalconf.Conf.DBType = config.Conf.OvalDBType
 	ovalconf.Conf.DBPath = config.Conf.OvalDBPath
 	util.Log.Infof("open oval-dictionary db (%s): %s",
@@ -40,7 +40,7 @@ func (o DebianBase) fillCveInfoFromOvalDB(r *models.ScanResult) error {
 				}
 				affected, _ := ver.NewVersion(p.Version)
 				if current.LessThan(affected) {
-					o.fillOvalInfo(r, &def)
+					o.update(r, &def)
 				}
 			}
 		}
@@ -48,7 +48,7 @@ func (o DebianBase) fillCveInfoFromOvalDB(r *models.ScanResult) error {
 	return nil
 }

-func (o DebianBase) fillOvalInfo(r *models.ScanResult, definition *ovalmodels.Definition) {
+func (o DebianBase) update(r *models.ScanResult, definition *ovalmodels.Definition) {
 	ovalContent := *o.convertToModel(definition)
 	ovalContent.Type = models.NewCveContentType(r.Family)
 	vinfo, ok := r.ScannedCves[definition.Debian.CveID]
@@ -103,15 +103,26 @@ type Debian struct {
 }

 // NewDebian creates OVAL client for Debian
-func NewDebian() *Debian {
-	return &Debian{}
+func NewDebian() Debian {
+	return Debian{}
 }

-// FillCveInfoFromOvalDB returns scan result after updating CVE info by OVAL
-func (o Debian) FillCveInfoFromOvalDB(r *models.ScanResult) error {
-	if err := o.fillCveInfoFromOvalDB(r); err != nil {
-		return err
+// FillWithOval returns scan result after updating CVE info by OVAL
+func (o Debian) FillWithOval(r *models.ScanResult) error {
+	if config.Conf.OvalDBURL != "" {
+		defs, err := getDefsByPackNameViaHTTP(r)
+		if err != nil {
+			return err
+		}
+		for _, def := range defs {
+			o.update(r, &def)
+		}
+	} else {
+		if err := o.fillFromOvalDB(r); err != nil {
+			return err
+		}
 	}
+
 	for _, vuln := range r.ScannedCves {
 		if cont, ok := vuln.CveContents[models.Debian]; ok {
 			cont.SourceLink = "https://security-tracker.debian.org/tracker/" + cont.CveID
@@ -127,13 +138,13 @@ type Ubuntu struct {
 }

 // NewUbuntu creates OVAL client for Debian
-func NewUbuntu() *Ubuntu {
-	return &Ubuntu{}
+func NewUbuntu() Ubuntu {
+	return Ubuntu{}
 }

-// FillCveInfoFromOvalDB returns scan result after updating CVE info by OVAL
-func (o Ubuntu) FillCveInfoFromOvalDB(r *models.ScanResult) error {
-	if err := o.fillCveInfoFromOvalDB(r); err != nil {
+// FillWithOval returns scan result after updating CVE info by OVAL
+func (o Ubuntu) FillWithOval(r *models.ScanResult) error {
+	if err := o.fillFromOvalDB(r); err != nil {
 		return err
 	}
 	for _, vuln := range r.ScannedCves {
--- a/oval/oval.go
+++ b/oval/oval.go
@@ -1,13 +1,132 @@
 package oval

 import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/cenkalti/backoff"
+	"github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/util"
+	ver "github.com/knqyf263/go-deb-version"
 	ovalmodels "github.com/kotakanbe/goval-dictionary/models"
+	"github.com/parnurzeal/gorequest"
 )

 // Client is the interface of OVAL client.
 type Client interface {
-	FillCveInfoFromOvalDB(r *models.ScanResult) error
+	FillWithOval(r *models.ScanResult) error
+}
+
+type request struct {
+	pack models.Package
+}
+
+type response struct {
+	pack *models.Package
+	defs []ovalmodels.Definition
+}
+
+// getDefsByPackNameViaHTTP fetches OVAL information via HTTP
+func getDefsByPackNameViaHTTP(r *models.ScanResult) (
+	relatedDefs []ovalmodels.Definition, err error) {
+
+	//TODO Health Check
+	reqChan := make(chan request, len(r.Packages))
+	resChan := make(chan response, len(r.Packages))
+	errChan := make(chan error, len(r.Packages))
+	defer close(reqChan)
+	defer close(resChan)
+	defer close(errChan)
+
+	go func() {
+		for _, pack := range r.Packages {
+			reqChan <- request{
+				pack: pack,
+			}
+		}
+	}()
+
+	concurrency := 10
+	tasks := util.GenWorkers(concurrency)
+	for range r.Packages {
+		tasks <- func() {
+			select {
+			case req := <-reqChan:
+				url, err := util.URLPathJoin(
+					config.Conf.OvalDBURL,
+					"packs",
+					r.Family,
+					r.Release,
+					req.pack.Name,
+				)
+				if err != nil {
+					errChan <- err
+				} else {
+					util.Log.Debugf("HTTP Request to %s", url)
+					httpGet(url, &req.pack, resChan, errChan)
+				}
+			}
+		}
+	}
+
+	timeout := time.After(2 * 60 * time.Second)
+	var errs []error
+	for range r.Packages {
+		select {
+		case res := <-resChan:
+			current, _ := ver.NewVersion(fmt.Sprintf("%s-%s",
+				res.pack.Version, res.pack.Release))
+			for _, def := range res.defs {
+				for _, p := range def.AffectedPacks {
+					affected, _ := ver.NewVersion(p.Version)
+					if res.pack.Name != p.Name || !current.LessThan(affected) {
+						continue
+					}
+					relatedDefs = append(relatedDefs, def)
+				}
+			}
+		case err := <-errChan:
+			errs = append(errs, err)
+		case <-timeout:
+			return nil, fmt.Errorf("Timeout Fetching OVAL")
+		}
+	}
+	if len(errs) != 0 {
+		return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs)
+	}
+	return
+}
+
+func httpGet(url string, pack *models.Package, resChan chan<- response, errChan chan<- error) {
+	var body string
+	var errs []error
+	var resp *http.Response
+	f := func() (err error) {
+		//  resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
+		resp, body, errs = gorequest.New().Get(url).End()
+		if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
+			return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", errs, url, resp)
+		}
+		return nil
+	}
+	notify := func(err error, t time.Duration) {
+		util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
+	}
+	err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
+	if err != nil {
+		errChan <- fmt.Errorf("HTTP Error %s", err)
+	}
+	defs := []ovalmodels.Definition{}
+	if err := json.Unmarshal([]byte(body), &defs); err != nil {
+		errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err)
+	}
+	resChan <- response{
+		pack: pack,
+		defs: defs,
+	}
 }

 func getPackages(r *models.ScanResult, d *ovalmodels.Definition) (names []string) {
--- a/oval/redhat.go
+++ b/oval/redhat.go
@@ -17,11 +17,22 @@ import (
 // RedHatBase is the base struct for RedHat and CentOS
 type RedHatBase struct{}

-// FillCveInfoFromOvalDB returns scan result after updating CVE info by OVAL
-func (o RedHatBase) FillCveInfoFromOvalDB(r *models.ScanResult) error {
-	if err := o.fillCveInfoFromOvalDB(r); err != nil {
-		return err
+// FillWithOval returns scan result after updating CVE info by OVAL
+func (o RedHatBase) FillWithOval(r *models.ScanResult) error {
+	if config.Conf.OvalDBURL != "" {
+		defs, err := getDefsByPackNameViaHTTP(r)
+		if err != nil {
+			return err
+		}
+		for _, def := range defs {
+			o.update(r, &def)
+		}
+	} else {
+		if err := o.fillFromOvalDB(r); err != nil {
+			return err
+		}
 	}
+
 	for _, vuln := range r.ScannedCves {
 		if cont, ok := vuln.CveContents[models.RedHat]; ok {
 			cont.SourceLink = "https://access.redhat.com/security/cve/" + cont.CveID
@@ -30,8 +41,21 @@ func (o RedHatBase) FillCveInfoFromOvalDB(r *models.ScanResult) error {
 	return nil
 }

-// FillCveInfoFromOvalDB returns scan result after updating CVE info by OVAL
-func (o RedHatBase) fillCveInfoFromOvalDB(r *models.ScanResult) error {
+// fillFromOvalDB returns scan result after updating CVE info by OVAL
+func (o RedHatBase) fillFromOvalDB(r *models.ScanResult) error {
+	defs, err := o.getDefsByPackNameFromOvalDB(r.Release, r.Packages)
+	if err != nil {
+		return err
+	}
+	for _, def := range defs {
+		o.update(r, &def)
+	}
+	return nil
+}
+
+func (o RedHatBase) getDefsByPackNameFromOvalDB(osRelease string,
+	packs models.Packages) (relatedDefs []ovalmodels.Definition, err error) {
+
 	ovalconf.Conf.DBType = config.Conf.OvalDBType
 	ovalconf.Conf.DBPath = config.Conf.OvalDBPath
 	util.Log.Infof("open oval-dictionary db (%s): %s",
@@ -39,28 +63,26 @@ func (o RedHatBase) fillCveInfoFromOvalDB(r *models.ScanResult) error {

 	d := db.NewRedHat()
 	defer d.Close()
-	for _, pack := range r.Packages {
-		definitions, err := d.GetByPackName(r.Release, pack.Name)
+	for _, pack := range packs {
+		definitions, err := d.GetByPackName(osRelease, pack.Name)
 		if err != nil {
-			return fmt.Errorf("Failed to get RedHat OVAL info by package name: %v", err)
+			return nil, fmt.Errorf("Failed to get RedHat OVAL info by package name: %v", err)
 		}
-		for _, definition := range definitions {
+		for _, def := range definitions {
 			current, _ := ver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release))
-			for _, p := range definition.AffectedPacks {
-				if pack.Name != p.Name {
+			for _, p := range def.AffectedPacks {
+				affected, _ := ver.NewVersion(p.Version)
+				if pack.Name != p.Name || !current.LessThan(affected) {
 					continue
 				}
-				affected, _ := ver.NewVersion(p.Version)
-				if current.LessThan(affected) {
-					o.fillOvalInfo(r, &definition)
-				}
+				relatedDefs = append(relatedDefs, def)
 			}
 		}
 	}
-	return nil
+	return
 }

-func (o RedHatBase) fillOvalInfo(r *models.ScanResult, definition *ovalmodels.Definition) {
+func (o RedHatBase) update(r *models.ScanResult, definition *ovalmodels.Definition) {
 	for _, cve := range definition.Advisory.Cves {
 		ovalContent := *o.convertToModel(cve.CveID, definition)
 		vinfo, ok := r.ScannedCves[cve.CveID]
@@ -77,7 +99,7 @@ func (o RedHatBase) fillOvalInfo(r *models.ScanResult, definition *ovalmodels.De
 			if _, ok := vinfo.CveContents[models.RedHat]; ok {
 				util.Log.Infof("%s will be updated by OVAL", cve.CveID)
 			} else {
-				util.Log.Infof("%s is also detected by OVAL", cve.CveID)
+				util.Log.Infof("%s also detected by OVAL", cve.CveID)
 				cveContents = models.CveContents{}
 			}

--- a/report/report.go
+++ b/report/report.go
@@ -90,18 +90,29 @@ func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, erro

 func fillCveInfo(r *models.ScanResult) error {
 	util.Log.Debugf("need to refresh")
-	if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDBURL == "" {
-		if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) {
-			return fmt.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s",
-				c.Conf.CveDBPath)
+	if c.Conf.CveDBType == "sqlite3" {
+		if c.Conf.CveDBURL == "" {
+			if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) {
+				return fmt.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s",
+					c.Conf.CveDBPath)
+			}
+		}
+		if c.Conf.OvalDBURL == "" {
+			if _, err := os.Stat(c.Conf.OvalDBPath); os.IsNotExist(err) {
+				//TODO Warning
+				return fmt.Errorf("SQLite3 DB(OVAL-Dictionary) is not exist: %s",
+					c.Conf.OvalDBPath)
+			}
 		}
 	}

-	if err := fillCveInfoFromOvalDB(r); err != nil {
+	util.Log.Debugf("Fill CVE detailed information with OVAL")
+	if err := fillWithOvalDB(r); err != nil {
 		return fmt.Errorf("Failed to fill OVAL information: %s", err)
 	}

-	if err := fillCveInfoFromCveDB(r); err != nil {
+	util.Log.Debugf("Fill CVE detailed information with CVE-DB")
+	if err := fillWithCveDB(r); err != nil {
 		return fmt.Errorf("Failed to fill CVE information: %s", err)
 	}

@@ -144,7 +155,7 @@ func fillCveDetail(r *models.ScanResult) error {
 	return nil
 }

-func fillCveInfoFromCveDB(r *models.ScanResult) error {
+func fillWithCveDB(r *models.ScanResult) error {
 	sInfo := c.Conf.Servers[r.ServerName]
 	if err := fillVulnByCpeNames(sInfo.CpeNames, r.ScannedCves); err != nil {
 		return err
@@ -155,7 +166,7 @@ func fillCveInfoFromCveDB(r *models.ScanResult) error {
 	return nil
 }

-func fillCveInfoFromOvalDB(r *models.ScanResult) error {
+func fillWithOvalDB(r *models.ScanResult) error {
 	var ovalClient oval.Client
 	switch r.Family {
 	case "debian":
@@ -172,7 +183,7 @@ func fillCveInfoFromOvalDB(r *models.ScanResult) error {
 	default:
 		return fmt.Errorf("Oval %s is not implemented yet", r.Family)
 	}
-	if err := ovalClient.FillCveInfoFromOvalDB(r); err != nil {
+	if err := ovalClient.FillWithOval(r); err != nil {
 		return err
 	}
 	return nil
--- a/report/tui.go
+++ b/report/tui.go
@@ -783,10 +783,12 @@ func detailLines() (string, error) {
 		}
 	}

+	summary := vinfo.CveContents.Summaries(r.Lang, r.Family)[0]
+
 	data := dataForTmpl{
 		CveID:      vinfo.CveID,
 		Cvsses:     append(vinfo.CveContents.Cvss3Scores(), vinfo.CveContents.Cvss2Scores()...),
-		Summary:    vinfo.CveContents.Summaries(r.Lang, r.Family)[0].Value,
+		Summary:    fmt.Sprintf("%s (%s)", summary.Value, summary.Type),
 		Confidence: vinfo.Confidence,
 		Cwes:       vinfo.CveContents.CweIDs(r.Family),
 		Links:      util.Distinct(links),