diff --git a/commands/report.go b/commands/report.go index ee2ea724..d19ab81a 100644 --- a/commands/report.go +++ b/commands/report.go @@ -45,12 +45,13 @@ type ReportCmd struct { ignoreUnscoredCves bool httpProxy string - cvedbtype string - cvedbpath string - cvedbURL string + cveDBType string + cveDBPath string + cveDBURL string - ovaldbtype string - ovaldbpath string + ovalDBType string + ovalDBPath string + ovalDBURL string toSlack bool toEMail bool @@ -98,6 +99,9 @@ func (*ReportCmd) Usage() string { [-cvedb-type=sqlite3|mysql|postgres] [-cvedb-path=/path/to/cve.sqlite3] [-cvedb-url=http://127.0.0.1:1323 or DB connection string] + [-ovaldb-type=sqlite3|mysql] + [-ovaldb-path=/path/to/oval.sqlite3] + [-ovaldb-url=http://127.0.0.1:1324 or DB connection string] [-cvss-over=7] [-diff] [-ignore-unscored-cves] @@ -152,36 +156,42 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) { "Refresh CVE information in JSON file under results dir") f.StringVar( - &p.cvedbtype, + &p.cveDBType, "cvedb-type", "sqlite3", "DB type for fetching CVE dictionary (sqlite3, mysql or postgres)") defaultCveDBPath := filepath.Join(wd, "cve.sqlite3") f.StringVar( - &p.cvedbpath, + &p.cveDBPath, "cvedb-path", defaultCveDBPath, "/path/to/sqlite3 (For get cve detail from cve.sqlite3)") f.StringVar( - &p.ovaldbtype, + &p.cveDBURL, + "cvedb-url", + "", + "http://cve-dictionary.com:1323 or mysql connection string") + + f.StringVar( + &p.ovalDBType, "ovaldb-type", "sqlite3", "DB type for fetching OVAL dictionary (sqlite3 or mysql)") defaultOvalDBPath := filepath.Join(wd, "oval.sqlite3") f.StringVar( - &p.ovaldbpath, + &p.ovalDBPath, "ovaldb-path", defaultOvalDBPath, "/path/to/sqlite3 (For get oval detail from oval.sqlite3)") f.StringVar( - &p.cvedbURL, - "cvedb-url", + &p.ovalDBURL, + "ovaldb-url", "", - "http://cve-dictionary.com:8080 or DB connection string") + "http://goval-dictionary.com:1324 or mysql connection string") f.Float64Var( &p.cvssScoreOver, @@ -290,11 +300,12 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} c.Conf.ResultsDir = p.resultsDir c.Conf.RefreshCve = p.refreshCve c.Conf.Diff = p.diff - c.Conf.CveDBType = p.cvedbtype - c.Conf.CveDBPath = p.cvedbpath - c.Conf.CveDBURL = p.cvedbURL - c.Conf.OvalDBType = p.ovaldbtype - c.Conf.OvalDBPath = p.ovaldbpath + c.Conf.CveDBType = p.cveDBType + c.Conf.CveDBPath = p.cveDBPath + c.Conf.CveDBURL = p.cveDBURL + c.Conf.OvalDBType = p.ovalDBType + c.Conf.OvalDBPath = p.ovalDBPath + c.Conf.OvalDBURL = p.ovalDBURL c.Conf.CvssScoreOver = p.cvssScoreOver c.Conf.IgnoreUnscoredCves = p.ignoreUnscoredCves c.Conf.HTTPProxy = p.httpProxy diff --git a/commands/tui.go b/commands/tui.go index ca2f002a..f40d03f7 100644 --- a/commands/tui.go +++ b/commands/tui.go @@ -38,12 +38,17 @@ type TuiCmd struct { configPath string logDir string - resultsDir string - refreshCve bool + resultsDir string + refreshCve bool + cvedbtype string cvedbpath string cveDictionaryURL string + ovalDBType string + ovalDBPath string + ovalDBURL string + pipe bool } @@ -61,6 +66,9 @@ func (*TuiCmd) Usage() string { [-cvedb-type=sqlite3|mysql|postgres] [-cvedb-path=/path/to/cve.sqlite3] [-cvedb-url=http://127.0.0.1:1323 or DB connection string] + [-ovaldb-type=sqlite3|mysql] + [-ovaldb-path=/path/to/oval.sqlite3] + [-ovaldb-url=http://127.0.0.1:1324 or DB connection string] [-refresh-cve] [-results-dir=/path/to/results] [-log-dir=/path/to/log] @@ -110,7 +118,26 @@ func (p *TuiCmd) SetFlags(f *flag.FlagSet) { &p.cveDictionaryURL, "cvedb-url", "", - "http://cve-dictionary.com:8080 or DB connection string") + "http://cve-dictionary.com:1323 or mysql connection string") + + f.StringVar( + &p.ovalDBType, + "ovaldb-type", + "sqlite3", + "DB type for fetching OVAL dictionary (sqlite3 or mysql)") + + defaultOvalDBPath := filepath.Join(wd, "oval.sqlite3") + f.StringVar( + &p.ovalDBPath, + "ovaldb-path", + defaultOvalDBPath, + "/path/to/sqlite3 (For get oval detail from oval.sqlite3)") + + f.StringVar( + &p.ovalDBURL, + "ovaldb-url", + "", + "http://goval-dictionary.com:1324 or mysql connection string") f.BoolVar( &p.pipe, @@ -139,6 +166,9 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s c.Conf.CveDBType = p.cvedbtype c.Conf.CveDBPath = p.cvedbpath c.Conf.CveDBURL = p.cveDictionaryURL + c.Conf.OvalDBType = p.ovalDBType + c.Conf.OvalDBPath = p.ovalDBPath + c.Conf.OvalDBURL = p.ovalDBURL log.Info("Validating config...") if !c.Conf.ValidateOnTui() { diff --git a/config/config.go b/config/config.go index 243efbe9..0ea115ad 100644 --- a/config/config.go +++ b/config/config.go @@ -53,13 +53,15 @@ type Config struct { LogDir string ResultsDir string - CveDBType string - CveDBPath string - CveDBURL string - CacheDBPath string + CveDBType string + CveDBPath string + CveDBURL string OvalDBType string OvalDBPath string + OvalDBURL string + + CacheDBPath string RefreshCve bool diff --git a/oval/debian.go b/oval/debian.go index 15d48ead..80f87f87 100644 --- a/oval/debian.go +++ b/oval/debian.go @@ -15,8 +15,8 @@ import ( // DebianBase is the base struct of Debian and Ubuntu type DebianBase struct{} -// fillCveInfoFromOvalDB returns scan result after updating CVE info by OVAL -func (o DebianBase) fillCveInfoFromOvalDB(r *models.ScanResult) error { +// fillFromOvalDB returns scan result after updating CVE info by OVAL +func (o DebianBase) fillFromOvalDB(r *models.ScanResult) error { ovalconf.Conf.DBType = config.Conf.OvalDBType ovalconf.Conf.DBPath = config.Conf.OvalDBPath util.Log.Infof("open oval-dictionary db (%s): %s", @@ -40,7 +40,7 @@ func (o DebianBase) fillCveInfoFromOvalDB(r *models.ScanResult) error { } affected, _ := ver.NewVersion(p.Version) if current.LessThan(affected) { - o.fillOvalInfo(r, &def) + o.update(r, &def) } } } @@ -48,7 +48,7 @@ func (o DebianBase) fillCveInfoFromOvalDB(r *models.ScanResult) error { return nil } -func (o DebianBase) fillOvalInfo(r *models.ScanResult, definition *ovalmodels.Definition) { +func (o DebianBase) update(r *models.ScanResult, definition *ovalmodels.Definition) { ovalContent := *o.convertToModel(definition) ovalContent.Type = models.NewCveContentType(r.Family) vinfo, ok := r.ScannedCves[definition.Debian.CveID] @@ -103,15 +103,26 @@ type Debian struct { } // NewDebian creates OVAL client for Debian -func NewDebian() *Debian { - return &Debian{} +func NewDebian() Debian { + return Debian{} } -// FillCveInfoFromOvalDB returns scan result after updating CVE info by OVAL -func (o Debian) FillCveInfoFromOvalDB(r *models.ScanResult) error { - if err := o.fillCveInfoFromOvalDB(r); err != nil { - return err +// FillWithOval returns scan result after updating CVE info by OVAL +func (o Debian) FillWithOval(r *models.ScanResult) error { + if config.Conf.OvalDBURL != "" { + defs, err := getDefsByPackNameViaHTTP(r) + if err != nil { + return err + } + for _, def := range defs { + o.update(r, &def) + } + } else { + if err := o.fillFromOvalDB(r); err != nil { + return err + } } + for _, vuln := range r.ScannedCves { if cont, ok := vuln.CveContents[models.Debian]; ok { cont.SourceLink = "https://security-tracker.debian.org/tracker/" + cont.CveID @@ -127,13 +138,13 @@ type Ubuntu struct { } // NewUbuntu creates OVAL client for Debian -func NewUbuntu() *Ubuntu { - return &Ubuntu{} +func NewUbuntu() Ubuntu { + return Ubuntu{} } -// FillCveInfoFromOvalDB returns scan result after updating CVE info by OVAL -func (o Ubuntu) FillCveInfoFromOvalDB(r *models.ScanResult) error { - if err := o.fillCveInfoFromOvalDB(r); err != nil { +// FillWithOval returns scan result after updating CVE info by OVAL +func (o Ubuntu) FillWithOval(r *models.ScanResult) error { + if err := o.fillFromOvalDB(r); err != nil { return err } for _, vuln := range r.ScannedCves { diff --git a/oval/oval.go b/oval/oval.go index e222e4b5..947a57f3 100644 --- a/oval/oval.go +++ b/oval/oval.go @@ -1,13 +1,132 @@ package oval import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/cenkalti/backoff" + "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + ver "github.com/knqyf263/go-deb-version" ovalmodels "github.com/kotakanbe/goval-dictionary/models" + "github.com/parnurzeal/gorequest" ) // Client is the interface of OVAL client. type Client interface { - FillCveInfoFromOvalDB(r *models.ScanResult) error + FillWithOval(r *models.ScanResult) error +} + +type request struct { + pack models.Package +} + +type response struct { + pack *models.Package + defs []ovalmodels.Definition +} + +// getDefsByPackNameViaHTTP fetches OVAL information via HTTP +func getDefsByPackNameViaHTTP(r *models.ScanResult) ( + relatedDefs []ovalmodels.Definition, err error) { + + //TODO Health Check + reqChan := make(chan request, len(r.Packages)) + resChan := make(chan response, len(r.Packages)) + errChan := make(chan error, len(r.Packages)) + defer close(reqChan) + defer close(resChan) + defer close(errChan) + + go func() { + for _, pack := range r.Packages { + reqChan <- request{ + pack: pack, + } + } + }() + + concurrency := 10 + tasks := util.GenWorkers(concurrency) + for range r.Packages { + tasks <- func() { + select { + case req := <-reqChan: + url, err := util.URLPathJoin( + config.Conf.OvalDBURL, + "packs", + r.Family, + r.Release, + req.pack.Name, + ) + if err != nil { + errChan <- err + } else { + util.Log.Debugf("HTTP Request to %s", url) + httpGet(url, &req.pack, resChan, errChan) + } + } + } + } + + timeout := time.After(2 * 60 * time.Second) + var errs []error + for range r.Packages { + select { + case res := <-resChan: + current, _ := ver.NewVersion(fmt.Sprintf("%s-%s", + res.pack.Version, res.pack.Release)) + for _, def := range res.defs { + for _, p := range def.AffectedPacks { + affected, _ := ver.NewVersion(p.Version) + if res.pack.Name != p.Name || !current.LessThan(affected) { + continue + } + relatedDefs = append(relatedDefs, def) + } + } + case err := <-errChan: + errs = append(errs, err) + case <-timeout: + return nil, fmt.Errorf("Timeout Fetching OVAL") + } + } + if len(errs) != 0 { + return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs) + } + return +} + +func httpGet(url string, pack *models.Package, resChan chan<- response, errChan chan<- error) { + var body string + var errs []error + var resp *http.Response + f := func() (err error) { + // resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End() + resp, body, errs = gorequest.New().Get(url).End() + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { + return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", errs, url, resp) + } + return nil + } + notify := func(err error, t time.Duration) { + util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err) + } + err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) + if err != nil { + errChan <- fmt.Errorf("HTTP Error %s", err) + } + defs := []ovalmodels.Definition{} + if err := json.Unmarshal([]byte(body), &defs); err != nil { + errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", body, err) + } + resChan <- response{ + pack: pack, + defs: defs, + } } func getPackages(r *models.ScanResult, d *ovalmodels.Definition) (names []string) { diff --git a/oval/redhat.go b/oval/redhat.go index edc4b3ee..e11d392d 100644 --- a/oval/redhat.go +++ b/oval/redhat.go @@ -17,11 +17,22 @@ import ( // RedHatBase is the base struct for RedHat and CentOS type RedHatBase struct{} -// FillCveInfoFromOvalDB returns scan result after updating CVE info by OVAL -func (o RedHatBase) FillCveInfoFromOvalDB(r *models.ScanResult) error { - if err := o.fillCveInfoFromOvalDB(r); err != nil { - return err +// FillWithOval returns scan result after updating CVE info by OVAL +func (o RedHatBase) FillWithOval(r *models.ScanResult) error { + if config.Conf.OvalDBURL != "" { + defs, err := getDefsByPackNameViaHTTP(r) + if err != nil { + return err + } + for _, def := range defs { + o.update(r, &def) + } + } else { + if err := o.fillFromOvalDB(r); err != nil { + return err + } } + for _, vuln := range r.ScannedCves { if cont, ok := vuln.CveContents[models.RedHat]; ok { cont.SourceLink = "https://access.redhat.com/security/cve/" + cont.CveID @@ -30,8 +41,21 @@ func (o RedHatBase) FillCveInfoFromOvalDB(r *models.ScanResult) error { return nil } -// FillCveInfoFromOvalDB returns scan result after updating CVE info by OVAL -func (o RedHatBase) fillCveInfoFromOvalDB(r *models.ScanResult) error { +// fillFromOvalDB returns scan result after updating CVE info by OVAL +func (o RedHatBase) fillFromOvalDB(r *models.ScanResult) error { + defs, err := o.getDefsByPackNameFromOvalDB(r.Release, r.Packages) + if err != nil { + return err + } + for _, def := range defs { + o.update(r, &def) + } + return nil +} + +func (o RedHatBase) getDefsByPackNameFromOvalDB(osRelease string, + packs models.Packages) (relatedDefs []ovalmodels.Definition, err error) { + ovalconf.Conf.DBType = config.Conf.OvalDBType ovalconf.Conf.DBPath = config.Conf.OvalDBPath util.Log.Infof("open oval-dictionary db (%s): %s", @@ -39,28 +63,26 @@ func (o RedHatBase) fillCveInfoFromOvalDB(r *models.ScanResult) error { d := db.NewRedHat() defer d.Close() - for _, pack := range r.Packages { - definitions, err := d.GetByPackName(r.Release, pack.Name) + for _, pack := range packs { + definitions, err := d.GetByPackName(osRelease, pack.Name) if err != nil { - return fmt.Errorf("Failed to get RedHat OVAL info by package name: %v", err) + return nil, fmt.Errorf("Failed to get RedHat OVAL info by package name: %v", err) } - for _, definition := range definitions { + for _, def := range definitions { current, _ := ver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release)) - for _, p := range definition.AffectedPacks { - if pack.Name != p.Name { + for _, p := range def.AffectedPacks { + affected, _ := ver.NewVersion(p.Version) + if pack.Name != p.Name || !current.LessThan(affected) { continue } - affected, _ := ver.NewVersion(p.Version) - if current.LessThan(affected) { - o.fillOvalInfo(r, &definition) - } + relatedDefs = append(relatedDefs, def) } } } - return nil + return } -func (o RedHatBase) fillOvalInfo(r *models.ScanResult, definition *ovalmodels.Definition) { +func (o RedHatBase) update(r *models.ScanResult, definition *ovalmodels.Definition) { for _, cve := range definition.Advisory.Cves { ovalContent := *o.convertToModel(cve.CveID, definition) vinfo, ok := r.ScannedCves[cve.CveID] @@ -77,7 +99,7 @@ func (o RedHatBase) fillOvalInfo(r *models.ScanResult, definition *ovalmodels.De if _, ok := vinfo.CveContents[models.RedHat]; ok { util.Log.Infof("%s will be updated by OVAL", cve.CveID) } else { - util.Log.Infof("%s is also detected by OVAL", cve.CveID) + util.Log.Infof("%s also detected by OVAL", cve.CveID) cveContents = models.CveContents{} } diff --git a/report/report.go b/report/report.go index c63397a1..5926b779 100644 --- a/report/report.go +++ b/report/report.go @@ -90,18 +90,29 @@ func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, erro func fillCveInfo(r *models.ScanResult) error { util.Log.Debugf("need to refresh") - if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDBURL == "" { - if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) { - return fmt.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s", - c.Conf.CveDBPath) + if c.Conf.CveDBType == "sqlite3" { + if c.Conf.CveDBURL == "" { + if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) { + return fmt.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s", + c.Conf.CveDBPath) + } + } + if c.Conf.OvalDBURL == "" { + if _, err := os.Stat(c.Conf.OvalDBPath); os.IsNotExist(err) { + //TODO Warning + return fmt.Errorf("SQLite3 DB(OVAL-Dictionary) is not exist: %s", + c.Conf.OvalDBPath) + } } } - if err := fillCveInfoFromOvalDB(r); err != nil { + util.Log.Debugf("Fill CVE detailed information with OVAL") + if err := fillWithOvalDB(r); err != nil { return fmt.Errorf("Failed to fill OVAL information: %s", err) } - if err := fillCveInfoFromCveDB(r); err != nil { + util.Log.Debugf("Fill CVE detailed information with CVE-DB") + if err := fillWithCveDB(r); err != nil { return fmt.Errorf("Failed to fill CVE information: %s", err) } @@ -144,7 +155,7 @@ func fillCveDetail(r *models.ScanResult) error { return nil } -func fillCveInfoFromCveDB(r *models.ScanResult) error { +func fillWithCveDB(r *models.ScanResult) error { sInfo := c.Conf.Servers[r.ServerName] if err := fillVulnByCpeNames(sInfo.CpeNames, r.ScannedCves); err != nil { return err @@ -155,7 +166,7 @@ func fillCveInfoFromCveDB(r *models.ScanResult) error { return nil } -func fillCveInfoFromOvalDB(r *models.ScanResult) error { +func fillWithOvalDB(r *models.ScanResult) error { var ovalClient oval.Client switch r.Family { case "debian": @@ -172,7 +183,7 @@ func fillCveInfoFromOvalDB(r *models.ScanResult) error { default: return fmt.Errorf("Oval %s is not implemented yet", r.Family) } - if err := ovalClient.FillCveInfoFromOvalDB(r); err != nil { + if err := ovalClient.FillWithOval(r); err != nil { return err } return nil diff --git a/report/tui.go b/report/tui.go index ef3a32fb..e2951a76 100644 --- a/report/tui.go +++ b/report/tui.go @@ -783,10 +783,12 @@ func detailLines() (string, error) { } } + summary := vinfo.CveContents.Summaries(r.Lang, r.Family)[0] + data := dataForTmpl{ CveID: vinfo.CveID, Cvsses: append(vinfo.CveContents.Cvss3Scores(), vinfo.CveContents.Cvss2Scores()...), - Summary: vinfo.CveContents.Summaries(r.Lang, r.Family)[0].Value, + Summary: fmt.Sprintf("%s (%s)", summary.Value, summary.Type), Confidence: vinfo.Confidence, Cwes: vinfo.CveContents.CweIDs(r.Family), Links: util.Distinct(links),