diff --git a/commands/util.go b/commands/util.go index 8f90624c..8c692632 100644 --- a/commands/util.go +++ b/commands/util.go @@ -193,9 +193,9 @@ func diff(curResults, preResults models.ScanResults) (diffed models.ScanResults, packages := models.Packages{} for _, s := range current.ScannedCves { - for _, pack := range s.Packages { - p := current.Packages[pack.Name] - packages[pack.Name] = p + for _, name := range s.PackageNames { + p := current.Packages[name] + packages[name] = p } } current.Packages = packages diff --git a/commands/util_test.go b/commands/util_test.go index 6732d716..3b6f4540 100644 --- a/commands/util_test.go +++ b/commands/util_test.go @@ -199,32 +199,14 @@ func TestDiff(t *testing.T) { Release: "16.04", ScannedCves: []models.VulnInfo{ { - CveID: "CVE-2012-6702", - Packages: models.Packages{ - "libexpat1": { - Name: "libexpat1", - Version: "2.1.0-7", - Release: "", - NewVersion: "2.1.0-7ubuntu0.16.04.2", - NewRelease: "", - Repository: "", - }, - }, + CveID: "CVE-2012-6702", + PackageNames: []string{"libexpat1"}, DistroAdvisories: []models.DistroAdvisory{}, CpeNames: []string{}, }, { - CveID: "CVE-2014-9761", - Packages: models.Packages{ - "libc-bin": { - Name: "libc-bin", - Version: "2.21-0ubuntu5", - Release: "", - NewVersion: "2.23-0ubuntu5", - NewRelease: "", - Repository: "", - }, - }, + CveID: "CVE-2014-9761", + PackageNames: []string{"libc-bin"}, DistroAdvisories: []models.DistroAdvisory{}, CpeNames: []string{}, }, @@ -242,32 +224,14 @@ func TestDiff(t *testing.T) { Release: "16.04", ScannedCves: []models.VulnInfo{ { - CveID: "CVE-2012-6702", - Packages: models.Packages{ - "libexpat1": { - Name: "libexpat1", - Version: "2.1.0-7", - Release: "", - NewVersion: "2.1.0-7ubuntu0.16.04.2", - NewRelease: "", - Repository: "", - }, - }, + CveID: "CVE-2012-6702", + PackageNames: []string{"libexpat1"}, DistroAdvisories: []models.DistroAdvisory{}, CpeNames: []string{}, }, { - CveID: "CVE-2014-9761", - Packages: models.Packages{ - "libc-bin": { - Name: "libc-bin", - Version: "2.21-0ubuntu5", - Release: "", - NewVersion: "2.23-0ubuntu5", - NewRelease: "", - Repository: "", - }, - }, + CveID: "CVE-2014-9761", + PackageNames: []string{"libc-bin"}, DistroAdvisories: []models.DistroAdvisory{}, CpeNames: []string{}, }, @@ -296,21 +260,26 @@ func TestDiff(t *testing.T) { Release: "16.04", ScannedCves: []models.VulnInfo{ { - CveID: "CVE-2016-6662", - Packages: models.Packages{ - "mysql-libs": { - Name: "mysql-libs", - Version: "5.1.73", - Release: "7.el6", - NewVersion: "5.1.73", - NewRelease: "8.el6_8", - Repository: "", - }, - }, + CveID: "CVE-2016-6662", + PackageNames: []string{"mysql-libs"}, DistroAdvisories: []models.DistroAdvisory{}, CpeNames: []string{}, }, }, + Packages: models.Packages{ + "mysql-libs": { + Name: "mysql-libs", + Version: "5.1.73", + Release: "7.el6", + NewVersion: "5.1.73", + NewRelease: "8.el6_8", + Repository: "", + Changelog: models.Changelog{ + Contents: "", + Method: "", + }, + }, + }, }, }, inPrevious: models.ScanResults{ @@ -329,17 +298,8 @@ func TestDiff(t *testing.T) { Release: "16.04", ScannedCves: []models.VulnInfo{ { - CveID: "CVE-2016-6662", - Packages: models.Packages{ - "mysql-libs": { - Name: "mysql-libs", - Version: "5.1.73", - Release: "7.el6", - NewVersion: "5.1.73", - NewRelease: "8.el6_8", - Repository: "", - }, - }, + CveID: "CVE-2016-6662", + PackageNames: []string{"mysql-libs"}, DistroAdvisories: []models.DistroAdvisory{}, CpeNames: []string{}, }, diff --git a/models/models.go b/models/models.go index c6923949..e18f8a65 100644 --- a/models/models.go +++ b/models/models.go @@ -377,7 +377,7 @@ func (v *VulnInfos) Upsert(vInfo VulnInfo) { type VulnInfo struct { CveID string Confidence Confidence - Packages Packages + PackageNames []string DistroAdvisories []DistroAdvisory // for Aamazon, RHEL, FreeBSD CpeNames []string CveContents CveContents @@ -391,8 +391,8 @@ func (v *VulnInfo) NilToEmpty() { if v.DistroAdvisories == nil { v.DistroAdvisories = []DistroAdvisory{} } - if v.Packages == nil { - v.Packages = Packages{} + if v.PackageNames == nil { + v.PackageNames = []string{} } if v.CveContents == nil { v.CveContents = NewCveContents() diff --git a/oval/debian.go b/oval/debian.go index c2b298d6..598badb4 100644 --- a/oval/debian.go +++ b/oval/debian.go @@ -67,10 +67,10 @@ func (o Debian) fillOvalInfo(r *models.ScanResult, definition *ovalmodels.Defini util.Log.Infof("%s is newly detected by OVAL", definition.Debian.CveID) vinfo = models.VulnInfo{ - CveID: definition.Debian.CveID, - Confidence: models.OvalMatch, - Packages: getPackages(r, definition), - CveContents: models.NewCveContents(ovalContent), + CveID: definition.Debian.CveID, + Confidence: models.OvalMatch, + PackageNames: getPackages(r, definition), + CveContents: models.NewCveContents(ovalContent), } } else { if _, ok := vinfo.CveContents.Get(models.NewCveContentType(r.Family)); !ok { diff --git a/oval/oval.go b/oval/oval.go index 557f464c..e222e4b5 100644 --- a/oval/oval.go +++ b/oval/oval.go @@ -10,12 +10,9 @@ type Client interface { FillCveInfoFromOvalDB(r *models.ScanResult) error } -func getPackages(r *models.ScanResult, d *ovalmodels.Definition) models.Packages { - packages := models.Packages{} +func getPackages(r *models.ScanResult, d *ovalmodels.Definition) (names []string) { for _, affectedPack := range d.AffectedPacks { - pack, _ := r.Packages[affectedPack.Name] - // pack.Changelog = models.Changelog{} - packages[affectedPack.Name] = pack + names = append(names, affectedPack.Name) } - return packages + return } diff --git a/oval/redhat.go b/oval/redhat.go index cf301599..ec2da4c8 100644 --- a/oval/redhat.go +++ b/oval/redhat.go @@ -63,10 +63,10 @@ func (o Redhat) fillOvalInfo(r *models.ScanResult, definition *ovalmodels.Defini if !ok { util.Log.Infof("%s is newly detected by OVAL", cve.CveID) vinfo = models.VulnInfo{ - CveID: cve.CveID, - Confidence: models.OvalMatch, - Packages: getPackages(r, definition), - CveContents: models.NewCveContents(ovalContent), + CveID: cve.CveID, + Confidence: models.OvalMatch, + PackageNames: getPackages(r, definition), + CveContents: models.NewCveContents(ovalContent), } } else { if _, ok := vinfo.CveContents.Get(models.RedHat); !ok { diff --git a/scan/debian.go b/scan/debian.go index 5db8cbac..fc915fd3 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -37,7 +37,14 @@ type debian struct { // NewDebian is constructor func newDebian(c config.ServerInfo) *debian { - d := &debian{} + d := &debian{ + base: base{ + osPackages: osPackages{ + Packages: models.Packages{}, + VulnInfos: models.VulnInfos{}, + }, + }, + } d.log = util.NewCustomLogger(c) d.setServerInfo(c) return d @@ -397,11 +404,20 @@ func (o *debian) parseAptGetUpgrade(stdout string) (upgradableNames []string, er return } +// DetectedCveID has CveID, Confidence and DetectionMethod fields +// LenientMatching will be true if this vulnerability is not detected by accurate version matching. +// see https://github.com/future-architect/vuls/pull/328 +type DetectedCveID struct { + CveID string + Confidence models.Confidence +} + func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta) (models.VulnInfos, error) { - resChan := make(chan struct { - models.Package - DetectedCveIDs - }, len(upgradablePacks)) + type response struct { + packName string + DetectedCveIDs []DetectedCveID + } + resChan := make(chan response, len(upgradablePacks)) errChan := make(chan error, len(upgradablePacks)) reqChan := make(chan models.Package, len(upgradablePacks)) defer close(resChan) @@ -415,7 +431,7 @@ func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta }() timeout := time.After(30 * 60 * time.Second) - concurrency := 1 + concurrency := 10 tasks := util.GenWorkers(concurrency) for range upgradablePacks { tasks <- func() { @@ -425,10 +441,7 @@ func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta changelog := o.getChangelogCache(meta, p) if 0 < len(changelog) { cveIDs, _ := o.getCveIDsFromChangelog(changelog, p.Name, p.Version) - resChan <- struct { - models.Package - DetectedCveIDs - }{p, cveIDs} + resChan <- response{p.Name, cveIDs} return } @@ -438,10 +451,7 @@ func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta if cveIDs, err := o.scanPackageCveIDs(p); err != nil { errChan <- err } else { - resChan <- struct { - models.Package - DetectedCveIDs - }{p, cveIDs} + resChan <- response{p.Name, cveIDs} } }(pack) } @@ -449,23 +459,23 @@ func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta } // { DetectedCveID{} : [package] } - cvePackages := make(map[DetectedCveID]models.Packages) + cvePackages := make(map[DetectedCveID][]string) errs := []error{} for i := 0; i < len(upgradablePacks); i++ { select { - case pair := <-resChan: - cves := pair.DetectedCveIDs + case response := <-resChan: + cves := response.DetectedCveIDs for _, cve := range cves { - packs, ok := cvePackages[cve] + packNames, ok := cvePackages[cve] if ok { - packs[cve.CveID] = pair.Package + packNames = append(packNames, response.packName) } else { - packs = models.Packages{} + packNames = []string{response.packName} } - cvePackages[cve] = packs + cvePackages[cve] = packNames } - o.log.Infof("(%d/%d) Scanned %s-%s : %s", - i+1, len(upgradablePacks), pair.Name, pair.Package.Version, cves) + o.log.Infof("(%d/%d) Scanned %s: %s", + i+1, len(upgradablePacks), response.packName, cves) case err := <-errChan: errs = append(errs, err) case <-timeout: @@ -482,11 +492,11 @@ func (o *debian) scanVulnInfos(upgradablePacks models.Packages, meta *cache.Meta } o.log.Debugf("%d Cves are found. cves: %v", len(cveIDs), cveIDs) var vinfos models.VulnInfos - for k, v := range cvePackages { + for cveID, names := range cvePackages { vinfos = append(vinfos, models.VulnInfo{ - CveID: k.CveID, - Confidence: k.Confidence, - Packages: v, + CveID: cveID.CveID, + Confidence: cveID.Confidence, + PackageNames: names, }) } @@ -615,6 +625,7 @@ func (o *debian) getCveIDsFromChangelog( Contents: "", Method: models.FailedToFindVersionInChangelog, } + //TODO Mutex o.Packages[name] = pack // If the version is not in changelog, return entire changelog to put into cache @@ -624,17 +635,6 @@ func (o *debian) getCveIDsFromChangelog( } } -// DetectedCveID has CveID, Confidence and DetectionMethod fields -// LenientMatching will be true if this vulnerability is not detected by accurate version matching. -// see https://github.com/future-architect/vuls/pull/328 -type DetectedCveID struct { - CveID string - Confidence models.Confidence -} - -// DetectedCveIDs is a slice of DetectedCveID -type DetectedCveIDs []DetectedCveID - var cveRe = regexp.MustCompile(`(CVE-\d{4}-\d{4,})`) // Collect CVE-IDs included in the changelog. diff --git a/scan/freebsd.go b/scan/freebsd.go index 9be3aad3..23cf06fa 100644 --- a/scan/freebsd.go +++ b/scan/freebsd.go @@ -33,7 +33,14 @@ type bsd struct { // NewBSD constructor func newBsd(c config.ServerInfo) *bsd { - d := &bsd{} + d := &bsd{ + base: base{ + osPackages: osPackages{ + Packages: models.Packages{}, + VulnInfos: models.VulnInfos{}, + }, + }, + } d.log = util.NewCustomLogger(c) d.setServerInfo(c) return d @@ -155,9 +162,13 @@ func (o *bsd) scanUnsecurePackages() (vulnInfos []models.VulnInfo, err error) { }) } + names := []string{} + for name := range packs { + names = append(names, name) + } vulnInfos = append(vulnInfos, models.VulnInfo{ CveID: k, - Packages: packs, + PackageNames: names, DistroAdvisories: disAdvs, Confidence: models.PkgAuditMatch, }) diff --git a/scan/redhat.go b/scan/redhat.go index 3411ac11..7c28b933 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -37,7 +37,14 @@ type redhat struct { // NewRedhat is constructor func newRedhat(c config.ServerInfo) *redhat { - r := &redhat{} + r := &redhat{ + base: base{ + osPackages: osPackages{ + Packages: models.Packages{}, + VulnInfos: models.VulnInfos{}, + }, + }, + } r.log = util.NewCustomLogger(c) r.setServerInfo(c) return r @@ -402,23 +409,32 @@ func (o *redhat) scanUnsecurePackagesUsingYumCheckUpdate() (models.VulnInfos, er // ] // - To // map { - // CveID: []models.Package + // CveID: models.Packages{} // } - cveIDPackMap := make(map[string][]models.Package) + cveIDPackages := make(map[string]models.Packages) for _, res := range results { for _, cveID := range res.CveIDs { - cveIDPackMap[cveID] = append( - cveIDPackMap[cveID], res.Package) + if packages, ok := cveIDPackages[cveID]; ok { + packages[res.Package.Name] = res.Package + cveIDPackages[cveID] = packages + } else { + cveIDPackages[cveID] = models.NewPackages(res.Package) + } } } vinfos := []models.VulnInfo{} - for k, v := range cveIDPackMap { + for cveID, packs := range cveIDPackages { + names := []string{} + for name := range packs { + names = append(names, name) + } + // Amazon, RHEL do not use this method, so VendorAdvisory do not set. vinfos = append(vinfos, models.VulnInfo{ - CveID: k, - Packages: models.NewPackages(v...), - Confidence: models.ChangelogExactMatch, + CveID: cveID, + PackageNames: names, + Confidence: models.ChangelogExactMatch, }) } return vinfos, nil @@ -732,17 +748,24 @@ func (o *redhat) scanUnsecurePackagesUsingYumPluginSecurity() (models.VulnInfos, vinfos[i].DistroAdvisories = advAppended packs := dict[advIDCveIDs.DistroAdvisory.AdvisoryID] - vinfos[i].Packages = vinfos[i].Packages.Merge(packs) + for _, pack := range packs { + vinfos[i].PackageNames = append(vinfos[i].PackageNames, pack.Name) + } found = true break } } if !found { + names := []string{} + packs := dict[advIDCveIDs.DistroAdvisory.AdvisoryID] + for _, pack := range packs { + names = append(names, pack.Name) + } cpinfo := models.VulnInfo{ CveID: cveID, DistroAdvisories: []models.DistroAdvisory{advIDCveIDs.DistroAdvisory}, - Packages: dict[advIDCveIDs.DistroAdvisory.AdvisoryID], + PackageNames: names, Confidence: models.YumUpdateSecurityMatch, } vinfos = append(vinfos, cpinfo)