diff --git a/Gopkg.lock b/Gopkg.lock index bc967282..a5359ee6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -4,8 +4,8 @@ [[projects]] name = "github.com/Azure/azure-sdk-for-go" packages = ["storage"] - revision = "59c277f1b488b81b1a5f944212f25b69bea8ece3" - version = "v10.1.0-beta" + revision = "5b6066bbd213e47c49a5fa2be2b29529bbcb6704" + version = "v10.1.1-beta" [[projects]] name = "github.com/Azure/go-autorest" @@ -28,8 +28,8 @@ [[projects]] name = "github.com/aws/aws-sdk-go" packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/shareddefaults","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"] - revision = "6d7fc1a00fcae6bbb53550f4a0b98324fd7aa250" - version = "v1.10.12" + revision = "6ff7be1a127941b8dc9a408ac9c2dd6eb2167a5d" + version = "v1.10.15" [[projects]] name = "github.com/boltdb/bolt" @@ -64,8 +64,8 @@ [[projects]] name = "github.com/go-redis/redis" packages = [".","internal","internal/consistenthash","internal/hashtag","internal/pool","internal/proto"] - revision = "da63fe7def48e378caf9539abf64b9b1e37bc01e" - version = "v6.5.3" + revision = "a005081ecd2d0d963a1a20efda049223205cf90a" + version = "v6.5.4" [[projects]] name = "github.com/go-sql-driver/mysql" @@ -149,7 +149,7 @@ branch = "master" name = "github.com/kotakanbe/goval-dictionary" packages = ["config","db","db/rdb","log","models"] - revision = "adf0b39cd7fea8f4493f7b65e7316179634be95d" + revision = "2c949ba2967dcd35574f2a78a12551c5326de6a9" [[projects]] branch = "master" @@ -239,7 +239,7 @@ branch = "master" name = "github.com/sirupsen/logrus" packages = ["."] - revision = "5ff5dd844dfeb4e23e27528f79f1f845bc8bb78f" + revision = "3eef8ce63d02f65d2da43214faf7bb19b0b2bb7a" [[projects]] branch = "master" @@ -263,19 +263,19 @@ branch = "master" name = "golang.org/x/crypto" packages = ["curve25519","ed25519","ed25519/internal/edwards25519","ssh","ssh/agent","ssh/terminal"] - revision = "7f7c0c2d75ebb4e32a21396ce36e87b6dadc91c9" + revision = "6914964337150723782436d56b3f21610a74ce7b" [[projects]] branch = "master" name = "golang.org/x/net" packages = ["context","idna","publicsuffix"] - revision = "b3756b4b77d7b13260a0a2ec658753cf48922eac" + revision = "ab5485076ff3407ad2d02db054635913f017b0ed" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "4cd6d1a821c7175768725b55ca82f14683a29ea4" + revision = "c4489faa6e5ab84c0ef40d6ee878f7a030281f0f" [[projects]] branch = "master" diff --git a/commands/report.go b/commands/report.go index 8fcf43db..7dfca1dd 100644 --- a/commands/report.go +++ b/commands/report.go @@ -410,7 +410,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } if c.Conf.OvalDBURL != "" { - err := oval.Base{}.CheckHealth() + err := oval.Base{}.CheckHTTPHealth() if err != nil { util.Log.Errorf("OVAL HTTP server is not running. err: %s", err) util.Log.Errorf("Run goval-dictionary as server mode before reporting or run with --ovaldb-path option") diff --git a/oval/debian.go b/oval/debian.go index ab5e4d78..ae4a07c3 100644 --- a/oval/debian.go +++ b/oval/debian.go @@ -25,7 +25,7 @@ func (o DebianBase) fillFromOvalDB(r *models.ScanResult) error { } else { ovalconf.Conf.DBPath = config.Conf.OvalDBURL } - util.Log.Infof("open oval-dictionary db (%s): %s", + util.Log.Infof("Open oval-dictionary db (%s): %s", ovalconf.Conf.DBType, ovalconf.Conf.DBPath) ovallog.Initialize(config.Conf.LogDir) @@ -68,7 +68,7 @@ func (o DebianBase) update(r *models.ScanResult, definition *ovalmodels.Definiti ovalContent.Type = models.NewCveContentType(r.Family) vinfo, ok := r.ScannedCves[definition.Debian.CveID] if !ok { - util.Log.Infof("%s is newly detected by OVAL", definition.Debian.CveID) + util.Log.Debugf("%s is newly detected by OVAL", definition.Debian.CveID) vinfo = models.VulnInfo{ CveID: definition.Debian.CveID, Confidence: models.OvalMatch, @@ -79,9 +79,9 @@ func (o DebianBase) update(r *models.ScanResult, definition *ovalmodels.Definiti cveContents := vinfo.CveContents ctype := models.NewCveContentType(r.Family) if _, ok := vinfo.CveContents[ctype]; ok { - util.Log.Infof("%s will be updated by OVAL", definition.Debian.CveID) + util.Log.Debugf("%s will be updated by OVAL", definition.Debian.CveID) } else { - util.Log.Infof("%s is also detected by OVAL", definition.Debian.CveID) + util.Log.Debugf("%s is also detected by OVAL", definition.Debian.CveID) cveContents = models.CveContents{} } if vinfo.Confidence.Score < models.OvalMatch.Score { diff --git a/oval/oval.go b/oval/oval.go index 174ba371..3c6f492d 100644 --- a/oval/oval.go +++ b/oval/oval.go @@ -11,21 +11,27 @@ import ( "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" ver "github.com/knqyf263/go-deb-version" + "github.com/kotakanbe/goval-dictionary/db" + ovallog "github.com/kotakanbe/goval-dictionary/log" ovalmodels "github.com/kotakanbe/goval-dictionary/models" "github.com/parnurzeal/gorequest" ) // Client is the interface of OVAL client. type Client interface { - CheckHealth() error + CheckHTTPHealth() error FillWithOval(r *models.ScanResult) error + + // CheckIfOvalFetched checks if oval entries are in DB by family, release. + CheckIfOvalFetched(string, string) (bool, error) + CheckIfOvalFresh(string, string) (bool, error) } // Base is a base struct type Base struct{} -// CheckHealth do health check -func (b Base) CheckHealth() error { +// CheckHTTPHealth do health check +func (b Base) CheckHTTPHealth() error { if !b.isFetchViaHTTP() { return nil } @@ -43,6 +49,82 @@ func (b Base) CheckHealth() error { return nil } +// CheckIfOvalFetched checks if oval entries are in DB by family, release. +func (b Base) CheckIfOvalFetched(osFamily, release string) (fetched bool, err error) { + ovallog.Initialize(config.Conf.LogDir) + if !b.isFetchViaHTTP() { + var ovaldb db.DB + if ovaldb, err = db.NewDB( + osFamily, + config.Conf.OvalDBType, + config.Conf.OvalDBPath, + config.Conf.DebugSQL, + ); err != nil { + return false, err + } + defer ovaldb.CloseDB() + count, err := ovaldb.CountDefs(osFamily, release) + if err != nil { + return false, fmt.Errorf("Failed to count OVAL defs: %s, %s, %v", + osFamily, release, err) + } + return 0 < count, nil + } + + url, _ := util.URLPathJoin(config.Conf.OvalDBURL, "count", osFamily, release) + resp, body, errs := gorequest.New().Get(url).End() + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { + return false, fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", + errs, url, resp) + } + count := 0 + if err := json.Unmarshal([]byte(body), &count); err != nil { + return false, fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", + body, err) + } + return 0 < count, nil +} + +// CheckIfOvalFresh checks if oval entries are fresh enough +func (b Base) CheckIfOvalFresh(osFamily, release string) (ok bool, err error) { + ovallog.Initialize(config.Conf.LogDir) + var lastModified time.Time + if !b.isFetchViaHTTP() { + var ovaldb db.DB + if ovaldb, err = db.NewDB( + osFamily, + config.Conf.OvalDBType, + config.Conf.OvalDBPath, + config.Conf.DebugSQL, + ); err != nil { + return false, err + } + defer ovaldb.CloseDB() + lastModified = ovaldb.GetLastModified(osFamily, release) + } else { + url, _ := util.URLPathJoin(config.Conf.OvalDBURL, "lastmodified", osFamily, release) + resp, body, errs := gorequest.New().Get(url).End() + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { + return false, fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", + errs, url, resp) + } + + if err := json.Unmarshal([]byte(body), &lastModified); err != nil { + return false, fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", + body, err) + } + } + + since := time.Now() + since = since.AddDate(0, 0, -3) + if lastModified.Before(since) { + util.Log.Warnf("%s-%s OVAL is old, last modified is %s. It's recommended to update OVAL to improve scanning accuracy. To update OVAL database, see https://github.com/kotakanbe/goval-dictionary#usage", + osFamily, release, lastModified) + return false, nil + } + return true, nil +} + func (b Base) isFetchViaHTTP() bool { // Default value of OvalDBType is sqlite3 if config.Conf.OvalDBURL != "" && config.Conf.OvalDBType == "sqlite3" { diff --git a/oval/redhat.go b/oval/redhat.go index d52c9c9b..7db9038c 100644 --- a/oval/redhat.go +++ b/oval/redhat.go @@ -8,7 +8,7 @@ import ( "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" - ver "github.com/knqyf263/go-deb-version" + ver "github.com/knqyf263/go-rpm-version" ovalconf "github.com/kotakanbe/goval-dictionary/config" db "github.com/kotakanbe/goval-dictionary/db" ovallog "github.com/kotakanbe/goval-dictionary/log" @@ -63,7 +63,7 @@ func (o RedHatBase) getDefsByPackNameFromOvalDB(osRelease string, } else { ovalconf.Conf.DBPath = config.Conf.OvalDBURL } - util.Log.Infof("open oval-dictionary db (%s): %s", + util.Log.Infof("Open oval-dictionary db (%s): %s", ovalconf.Conf.DBType, ovalconf.Conf.DBPath) ovallog.Initialize(config.Conf.LogDir) @@ -84,9 +84,9 @@ func (o RedHatBase) getDefsByPackNameFromOvalDB(osRelease string, return nil, fmt.Errorf("Failed to get RedHat OVAL info by package name: %v", err) } for _, def := range definitions { - current, _ := ver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release)) + current := ver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release)) for _, p := range def.AffectedPacks { - affected, _ := ver.NewVersion(p.Version) + affected := ver.NewVersion(p.Version) if pack.Name != p.Name || !current.LessThan(affected) { continue } diff --git a/report/cve_client.go b/report/cve_client.go index 52a2cfd2..19b5742f 100644 --- a/report/cve_client.go +++ b/report/cve_client.go @@ -147,7 +147,7 @@ func (api cvedictClient) FetchCveDetailsFromCveDB(cveIDs []string) (cveDetails [ return []*cve.CveDetail{}, fmt.Errorf("Failed to New DB. err: %s", err) } - log.Infof("Opening DB (%s).", driver.Name()) + util.Log.Infof("Opening DB (%s).", driver.Name()) if err := driver.OpenDB( cveconfig.Conf.DBType, cveconfig.Conf.DBPath, diff --git a/report/report.go b/report/report.go index dbf90310..9d268fb3 100644 --- a/report/report.go +++ b/report/report.go @@ -79,12 +79,12 @@ func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, erro func fillCveInfo(r *models.ScanResult) error { util.Log.Debugf("need to refresh") - util.Log.Debugf("Fill CVE detailed information with OVAL") + util.Log.Infof("Fill CVE detailed information with OVAL") if err := fillWithOval(r); err != nil { return fmt.Errorf("Failed to fill OVAL information: %s", err) } - util.Log.Debugf("Fill CVE detailed information with CVE-DB") + util.Log.Infof("Fill CVE detailed information with CVE-DB") if err := fillWithCveDB(r); err != nil { return fmt.Errorf("Failed to fill CVE information: %s", err) } @@ -139,23 +139,50 @@ func fillWithCveDB(r *models.ScanResult) error { return nil } -func fillWithOval(r *models.ScanResult) error { +func fillWithOval(r *models.ScanResult) (err error) { var ovalClient oval.Client + var ovalFamily string + + // TODO switch r.Family { case c.Debian: ovalClient = oval.NewDebian() + ovalFamily = c.Debian case c.Ubuntu: ovalClient = oval.NewUbuntu() + ovalFamily = c.Ubuntu case c.RedHat: ovalClient = oval.NewRedhat() + ovalFamily = c.RedHat case c.CentOS: ovalClient = oval.NewCentOS() + //use RedHat's OVAL + ovalFamily = c.RedHat + //TODO implement OracleLinux + // case c.Oracle: + // ovalClient = oval.New() + // ovalFamily = c.Oracle case c.Amazon, c.Oracle, c.Raspbian, c.FreeBSD: - //TODO implement OracleLinux return nil default: return fmt.Errorf("Oval %s is not implemented yet", r.Family) } + + ok, err := ovalClient.CheckIfOvalFetched(ovalFamily, r.Release) + if err != nil { + return err + } + if !ok { + util.Log.Warnf("OVAL is emtpy: %s-%s. It's recommended to use OVAL to improve scanning accuracy. To fetch OVAL database, see https://github.com/kotakanbe/goval-dictionary#usage", r.Family, r.Release) + return nil + } + + _, err = ovalClient.CheckIfOvalFresh(ovalFamily, r.Release) + if err != nil { + return err + } + util.Log.Infof("OVAL is fresh: %s-%s ", r.Family, r.Release) + if err := ovalClient.FillWithOval(r); err != nil { return err } diff --git a/report/tui.go b/report/tui.go index 9f762f15..5c2838dc 100644 --- a/report/tui.go +++ b/report/tui.go @@ -47,6 +47,10 @@ func RunTui(results models.ScanResults) subcommands.ExitStatus { // g, err := gocui.NewGui(gocui.OutputNormal) g := gocui.NewGui() + if err := g.Init(); err != nil { + log.Errorf("%s", err) + return subcommands.ExitFailure + } defer g.Close() g.SetLayout(layout) diff --git a/scan/redhat.go b/scan/redhat.go index f1960ae7..aa4e0fba 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -254,12 +254,6 @@ func (o *redhat) scanPackages() error { return nil } - //TODO Cache changelogs to bolt - //TODO --with-changelog - if err := o.fillChangelogs(updatable); err != nil { - return nil - } - var vinfos models.VulnInfos if vinfos, err = o.scanUnsecurePackages(updatable); err != nil { o.log.Errorf("Failed to scan vulnerable packages") @@ -379,6 +373,12 @@ func (o *redhat) parseUpdatablePacksLine(line string) (models.Package, error) { } func (o *redhat) scanUnsecurePackages(updatable models.Packages) (models.VulnInfos, error) { + //TODO Cache changelogs to bolt + //TODO --with-changelog + if err := o.fillChangelogs(updatable); err != nil { + return nil, err + } + if o.Distro.Family != config.CentOS { // Amazon, RHEL, Oracle Linux has yum updateinfo as default // yum updateinfo can collenct vendor advisory information.