From 5a0a6abf1101d6f0bdec0ffcea13b294935cc408 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Fri, 13 Oct 2017 17:22:11 +0900 Subject: [PATCH] Fix OVAL detection on Debian and Ubuntu (#509) * Add filter options to tui subcommand (#508) * Capture version of source packages on Debian based linux * Change makefile, gofmt -s * Refactoring * Implement OVAL detection of source packages for Debian, Ubuntu --- GNUmakefile | 8 +- models/models.go | 2 +- models/packages.go | 32 ++++- models/packages_test.go | 46 +++++++ models/scanresults.go | 6 +- oval/debian.go | 27 ++-- oval/debian_test.go | 8 +- oval/redhat.go | 10 +- oval/redhat_test.go | 7 +- oval/suse.go | 5 +- oval/util.go | 255 ++++++++++++++++++------------------ oval/util_test.go | 279 ++++++++++++++++++++++++++-------------- scan/base.go | 1 + scan/debian.go | 77 +++++++---- scan/debian_test.go | 33 ----- scan/redhat.go | 8 +- scan/serverapi.go | 3 + 17 files changed, 492 insertions(+), 315 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index d44cb749..7adf747f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -31,10 +31,10 @@ depup: go get -u github.com/golang/dep/... dep ensure -update -build: main.go dep +build: main.go dep pretest go build -ldflags "$(LDFLAGS)" -o vuls $< -install: main.go dep +install: main.go dep pretest go install -ldflags "$(LDFLAGS)" @@ -47,10 +47,10 @@ vet: echo $(PKGS) | xargs go vet || exit; fmt: - gofmt -w $(SRCS) + gofmt -s -w $(SRCS) fmtcheck: - $(foreach file,$(SRCS),gofmt -d $(file);) + $(foreach file,$(SRCS),gofmt -s -d $(file);) pretest: lint vet fmtcheck diff --git a/models/models.go b/models/models.go index 96850f24..705ac8d0 100644 --- a/models/models.go +++ b/models/models.go @@ -18,4 +18,4 @@ along with this program. If not, see . package models // JSONVersion is JSON Version -const JSONVersion = 2 +const JSONVersion = 3 diff --git a/models/packages.go b/models/packages.go index 473d52bd..48965a8f 100644 --- a/models/packages.go +++ b/models/packages.go @@ -81,7 +81,7 @@ func (ps Packages) FindOne(f func(Package) bool) (string, Package, bool) { return "", Package{}, false } -// Package has installed packages. +// Package has installed binary packages. type Package struct { Name string Version string @@ -116,6 +116,8 @@ func (p Package) FormatVersionFromTo(notFixedYet bool) string { to := p.FormatNewVer() if notFixedYet { to = "Not Fixed Yet" + } else if p.NewVersion == "" { + to = "Unknown" } return fmt.Sprintf("%s-%s -> %s", p.Name, p.FormatVer(), to) } @@ -151,3 +153,31 @@ type Changelog struct { Contents string Method DetectionMethod } + +// SrcPackage has installed source package information. +// Debian based Linux has both of package and source information in dpkg. +// OVAL database often includes a source version (Not a binary version), +// so it is also needed to capture source version for OVAL version comparison. +// https://github.com/future-architect/vuls/issues/504 +type SrcPackage struct { + Name string + Version string + BinaryNames []string +} + +// AddBinaryName add the name if not exists +func (s *SrcPackage) AddBinaryName(name string) { + found := false + for _, n := range s.BinaryNames { + if n == name { + return + } + } + if !found { + s.BinaryNames = append(s.BinaryNames, name) + } +} + +// SrcPackages is Map of SrcPackage +// { "package-name": SrcPackage } +type SrcPackages map[string]SrcPackage diff --git a/models/packages_test.go b/models/packages_test.go index 48eb99d8..2c74a2e8 100644 --- a/models/packages_test.go +++ b/models/packages_test.go @@ -87,3 +87,49 @@ func TestMerge(t *testing.T) { t.Errorf("expected %s, actual %s", e, a) } } + +func TestAddBinaryName(t *testing.T) { + var tests = []struct { + in SrcPackage + name string + expected SrcPackage + }{ + { + SrcPackage{Name: "hoge"}, + "curl", + SrcPackage{ + Name: "hoge", + BinaryNames: []string{"curl"}, + }, + }, + { + SrcPackage{ + Name: "hoge", + BinaryNames: []string{"curl"}, + }, + "curl", + SrcPackage{ + Name: "hoge", + BinaryNames: []string{"curl"}, + }, + }, + { + SrcPackage{ + Name: "hoge", + BinaryNames: []string{"curl"}, + }, + "openssh", + SrcPackage{ + Name: "hoge", + BinaryNames: []string{"curl", "openssh"}, + }, + }, + } + + for _, tt := range tests { + tt.in.AddBinaryName(tt.name) + if !reflect.DeepEqual(tt.in, tt.expected) { + t.Errorf("expected %#v, actual %#v", tt.in, tt.expected) + } + } +} diff --git a/models/scanresults.go b/models/scanresults.go index c144d139..60df1661 100644 --- a/models/scanresults.go +++ b/models/scanresults.go @@ -46,8 +46,10 @@ type ScanResult struct { RunningKernel Kernel Packages Packages - Errors []string - Optional [][]interface{} + SrcPackages SrcPackages + + Errors []string + Optional [][]interface{} Config struct { Scan config.Config diff --git a/oval/debian.go b/oval/debian.go index 873a824d..7dd7f5da 100644 --- a/oval/debian.go +++ b/oval/debian.go @@ -60,8 +60,10 @@ func (o DebianBase) update(r *models.ScanResult, defPacks defPacks) { // uniq(vinfo.PackNames + defPacks.actuallyAffectedPackNames) for _, pack := range vinfo.AffectedPackages { - defPacks.actuallyAffectedPackNames[pack.Name] = true + notFixedYet, _ := defPacks.actuallyAffectedPackNames[pack.Name] + defPacks.actuallyAffectedPackNames[pack.Name] = notFixedYet } + vinfo.AffectedPackages = defPacks.toPackStatuses(r.Family, r.Packages) vinfo.AffectedPackages.Sort() r.ScannedCves[defPacks.def.Debian.CveID] = vinfo @@ -107,11 +109,17 @@ func (o Debian) FillWithOval(r *models.ScanResult) (err error) { //Debian's uname gives both of kernel release(uname -r), version(kernel-image version) linuxImage := "linux-image-" + r.RunningKernel.Release + // Add linux and set the version of running kernel to search OVAL. + newVer := "" + if p, ok := r.Packages[linuxImage]; ok { + newVer = p.NewVersion + } if r.Container.ContainerID == "" { r.Packages["linux"] = models.Package{ - Name: "linux", - Version: r.RunningKernel.Version, + Name: "linux", + Version: r.RunningKernel.Version, + NewVersion: newVer, } } @@ -121,7 +129,7 @@ func (o Debian) FillWithOval(r *models.ScanResult) (err error) { return err } } else { - if relatedDefs, err = getDefsByPackNameFromOvalDB(o.family, r.Release, r.Packages); err != nil { + if relatedDefs, err = getDefsByPackNameFromOvalDB(r); err != nil { return err } } @@ -129,10 +137,10 @@ func (o Debian) FillWithOval(r *models.ScanResult) (err error) { delete(r.Packages, "linux") for _, defPacks := range relatedDefs.entries { - // Remove linux added above to search for oval + // Remove "linux" added above for oval search // linux is not a real package name (key of affected packages in OVAL) - if _, ok := defPacks.actuallyAffectedPackNames["linux"]; ok { - defPacks.actuallyAffectedPackNames[linuxImage] = true + if notFixedYet, ok := defPacks.actuallyAffectedPackNames["linux"]; ok { + defPacks.actuallyAffectedPackNames[linuxImage] = notFixedYet delete(defPacks.actuallyAffectedPackNames, "linux") for i, p := range defPacks.def.AffectedPacks { if p.Name == "linux" { @@ -141,6 +149,7 @@ func (o Debian) FillWithOval(r *models.ScanResult) (err error) { } } } + o.update(r, defPacks) } @@ -230,7 +239,7 @@ func (o Ubuntu) FillWithOval(r *models.ScanResult) (err error) { return err } } else { - if relatedDefs, err = getDefsByPackNameFromOvalDB(o.family, r.Release, r.Packages); err != nil { + if relatedDefs, err = getDefsByPackNameFromOvalDB(r); err != nil { return err } } @@ -240,7 +249,6 @@ func (o Ubuntu) FillWithOval(r *models.ScanResult) (err error) { } for _, defPacks := range relatedDefs.entries { - // Remove "linux" added above to search for oval // "linux" is not a real package name (key of affected packages in OVAL) if _, ok := defPacks.actuallyAffectedPackNames["linux"]; !found && ok { @@ -253,6 +261,7 @@ func (o Ubuntu) FillWithOval(r *models.ScanResult) (err error) { } } } + o.update(r, defPacks) } diff --git a/oval/debian_test.go b/oval/debian_test.go index f522d1c6..c39826e7 100644 --- a/oval/debian_test.go +++ b/oval/debian_test.go @@ -36,7 +36,10 @@ func TestPackNamesOfUpdateDebian(t *testing.T) { in: models.ScanResult{ ScannedCves: models.VulnInfos{ "CVE-2000-1000": models.VulnInfo{ - AffectedPackages: models.PackageStatuses{{Name: "packA"}}, + AffectedPackages: models.PackageStatuses{ + {Name: "packA"}, + {Name: "packC"}, + }, }, }, }, @@ -55,7 +58,8 @@ func TestPackNamesOfUpdateDebian(t *testing.T) { "CVE-2000-1000": models.VulnInfo{ AffectedPackages: models.PackageStatuses{ {Name: "packA"}, - {Name: "packB"}, + {Name: "packB", NotFixedYet: true}, + {Name: "packC"}, }, }, }, diff --git a/oval/redhat.go b/oval/redhat.go index cd5c28f5..42385bf1 100644 --- a/oval/redhat.go +++ b/oval/redhat.go @@ -41,8 +41,7 @@ func (o RedHatBase) FillWithOval(r *models.ScanResult) (err error) { return err } } else { - if relatedDefs, err = getDefsByPackNameFromOvalDB( - o.family, r.Release, r.Packages); err != nil { + if relatedDefs, err = getDefsByPackNameFromOvalDB(r); err != nil { return err } } @@ -98,7 +97,8 @@ func (o RedHatBase) update(r *models.ScanResult, defPacks defPacks) { // uniq(vinfo.PackNames + defPacks.actuallyAffectedPackNames) for _, pack := range vinfo.AffectedPackages { - defPacks.actuallyAffectedPackNames[pack.Name] = true + notFixedYet, _ := defPacks.actuallyAffectedPackNames[pack.Name] + defPacks.actuallyAffectedPackNames[pack.Name] = notFixedYet } vinfo.AffectedPackages = defPacks.toPackStatuses(r.Family, r.Packages) vinfo.AffectedPackages.Sort() @@ -156,7 +156,7 @@ func (o RedHatBase) parseCvss2(scoreVector string) (score float64, vector string if score, err = strconv.ParseFloat(ss[0], 64); err != nil { return 0, "" } - return score, strings.Join(ss[1:len(ss)], "/") + return score, strings.Join(ss[1:], "/") } return 0, "" } @@ -170,7 +170,7 @@ func (o RedHatBase) parseCvss3(scoreVector string) (score float64, vector string if score, err = strconv.ParseFloat(ss[0], 64); err != nil { return 0, "" } - return score, strings.Join(ss[1:len(ss)], "/") + return score, strings.Join(ss[1:], "/") } return 0, "" } diff --git a/oval/redhat_test.go b/oval/redhat_test.go index 7914686c..97559a25 100644 --- a/oval/redhat_test.go +++ b/oval/redhat_test.go @@ -102,7 +102,10 @@ func TestPackNamesOfUpdate(t *testing.T) { in: models.ScanResult{ ScannedCves: models.VulnInfos{ "CVE-2000-1000": models.VulnInfo{ - AffectedPackages: models.PackageStatuses{{Name: "packA"}}, + AffectedPackages: models.PackageStatuses{ + {Name: "packA"}, + {Name: "packB", NotFixedYet: false}, + }, }, }, }, @@ -125,7 +128,7 @@ func TestPackNamesOfUpdate(t *testing.T) { "CVE-2000-1000": models.VulnInfo{ AffectedPackages: models.PackageStatuses{ {Name: "packA"}, - {Name: "packB"}, + {Name: "packB", NotFixedYet: true}, }, }, }, diff --git a/oval/suse.go b/oval/suse.go index b0078de7..312c87e3 100644 --- a/oval/suse.go +++ b/oval/suse.go @@ -47,7 +47,7 @@ func (o SUSE) FillWithOval(r *models.ScanResult) (err error) { return err } } else { - if relatedDefs, err = getDefsByPackNameFromOvalDB(o.family, r.Release, r.Packages); err != nil { + if relatedDefs, err = getDefsByPackNameFromOvalDB(r); err != nil { return err } } @@ -93,7 +93,8 @@ func (o SUSE) update(r *models.ScanResult, defPacks defPacks) { // uniq(vinfo.PackNames + defPacks.actuallyAffectedPackNames) for _, pack := range vinfo.AffectedPackages { - defPacks.actuallyAffectedPackNames[pack.Name] = true + notFixedYet, _ := defPacks.actuallyAffectedPackNames[pack.Name] + defPacks.actuallyAffectedPackNames[pack.Name] = notFixedYet } vinfo.AffectedPackages = defPacks.toPackStatuses(r.Family, r.Packages) vinfo.AffectedPackages.Sort() diff --git a/oval/util.go b/oval/util.go index aa537744..a2f4cc71 100644 --- a/oval/util.go +++ b/oval/util.go @@ -40,105 +40,48 @@ type ovalResult struct { } type defPacks struct { - def ovalmodels.Definition + def ovalmodels.Definition + + // BinaryPackageName : NotFixedYet actuallyAffectedPackNames map[string]bool } func (e defPacks) toPackStatuses(family string, packs models.Packages) (ps models.PackageStatuses) { - switch family { - case config.Ubuntu: - packNotFixedYet := map[string]bool{} - for _, p := range e.def.AffectedPacks { - packNotFixedYet[p.Name] = p.NotFixedYet - } - for k := range e.actuallyAffectedPackNames { - ps = append(ps, models.PackageStatus{ - Name: k, - NotFixedYet: packNotFixedYet[k], - }) - } - - case config.CentOS, config.Debian: - // There are many packages that has been fixed in RedHat, but not been fixed in CentOS - for name := range e.actuallyAffectedPackNames { - pack, ok := packs[name] - if !ok { - util.Log.Warnf("Failed to find in Package list: %s", name) - return - } - - ovalPackVer := "" - for _, p := range e.def.AffectedPacks { - if p.Name == name { - ovalPackVer = p.Version - break - } - } - if ovalPackVer == "" { - util.Log.Warnf("Failed to find in Oval Package list: %s", name) - return - } - - if pack.NewVersion == "" { - // compare version: installed vs oval - vera := rpmver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release)) - verb := rpmver.NewVersion(ovalPackVer) - notFixedYet := false - if vera.LessThan(verb) { - notFixedYet = true - } - ps = append(ps, models.PackageStatus{ - Name: name, - NotFixedYet: notFixedYet, - }) - } else { - // compare version: newVer vs oval - packNewVer := fmt.Sprintf("%s-%s", pack.NewVersion, pack.NewRelease) - vera := rpmver.NewVersion(packNewVer) - verb := rpmver.NewVersion(ovalPackVer) - notFixedYet := false - if vera.LessThan(verb) { - notFixedYet = true - } - ps = append(ps, models.PackageStatus{ - Name: name, - NotFixedYet: notFixedYet, - }) - } - } - - default: - for k := range e.actuallyAffectedPackNames { - ps = append(ps, models.PackageStatus{ - Name: k, - }) - } + for name, notFixedYet := range e.actuallyAffectedPackNames { + ps = append(ps, models.PackageStatus{ + Name: name, + NotFixedYet: notFixedYet, + }) } - return } -func (e *ovalResult) upsert(def ovalmodels.Definition, packName string) (upserted bool) { +func (e *ovalResult) upsert(def ovalmodels.Definition, packName string, notFixedYet bool) (upserted bool) { for i, entry := range e.entries { if entry.def.DefinitionID == def.DefinitionID { - e.entries[i].actuallyAffectedPackNames[packName] = true + e.entries[i].actuallyAffectedPackNames[packName] = notFixedYet return true } } e.entries = append(e.entries, defPacks{ def: def, - actuallyAffectedPackNames: map[string]bool{packName: true}, + actuallyAffectedPackNames: map[string]bool{packName: notFixedYet}, }) + return false } type request struct { - pack models.Package + packName string + versionRelease string + NewVersionRelease string + binaryPackNames []string + isSrcPack bool } type response struct { - pack *models.Package - defs []ovalmodels.Definition + request request + defs []ovalmodels.Definition } // getDefsByPackNameViaHTTP fetches OVAL information via HTTP @@ -152,17 +95,32 @@ func getDefsByPackNameViaHTTP(r *models.ScanResult) ( defer close(resChan) defer close(errChan) + names := []string{} go func() { for _, pack := range r.Packages { + names = append(names, pack.Name) reqChan <- request{ - pack: pack, + packName: pack.Name, + versionRelease: pack.FormatVer(), + NewVersionRelease: pack.FormatVer(), + isSrcPack: false, } + for _, pack := range r.SrcPackages { + names = append(names, pack.Name) + reqChan <- request{ + packName: pack.Name, + binaryPackNames: pack.BinaryNames, + versionRelease: pack.Version, + isSrcPack: true, + } + } + } }() concurrency := 10 tasks := util.GenWorkers(concurrency) - for range r.Packages { + for range names { tasks <- func() { select { case req := <-reqChan: @@ -171,13 +129,13 @@ func getDefsByPackNameViaHTTP(r *models.ScanResult) ( "packs", r.Family, r.Release, - req.pack.Name, + req.packName, ) if err != nil { errChan <- err } else { util.Log.Debugf("HTTP Request to %s", url) - httpGet(url, &req.pack, resChan, errChan) + httpGet(url, req, resChan, errChan) } } } @@ -185,26 +143,21 @@ func getDefsByPackNameViaHTTP(r *models.ScanResult) ( timeout := time.After(2 * 60 * time.Second) var errs []error - for range r.Packages { + for range names { select { case res := <-resChan: for _, def := range res.defs { - for _, p := range def.AffectedPacks { - if res.pack.Name != p.Name { - continue - } + affected, notFixedYet := isOvalDefAffected(def, r.Family, res.request) + if !affected { + continue + } - if p.NotFixedYet { - relatedDefs.upsert(def, p.Name) - continue - } - - if less, err := lessThan(r.Family, *res.pack, p); err != nil { - util.Log.Debugf("Failed to parse versions: %s", err) - util.Log.Debugf("%#v\n%#v", *res.pack, p) - } else if less { - relatedDefs.upsert(def, p.Name) + if res.request.isSrcPack { + for _, n := range res.request.binaryPackNames { + relatedDefs.upsert(def, n, false) } + } else { + relatedDefs.upsert(def, res.request.packName, notFixedYet) } } case err := <-errChan: @@ -219,7 +172,7 @@ func getDefsByPackNameViaHTTP(r *models.ScanResult) ( return } -func httpGet(url string, pack *models.Package, resChan chan<- response, errChan chan<- error) { +func httpGet(url string, req request, resChan chan<- response, errChan chan<- error) { var body string var errs []error var resp *http.Response @@ -257,14 +210,12 @@ func httpGet(url string, pack *models.Package, resChan chan<- response, errChan return } resChan <- response{ - pack: pack, - defs: defs, + request: req, + defs: defs, } } -func getDefsByPackNameFromOvalDB(family, osRelease string, - installedPacks models.Packages) (relatedDefs ovalResult, err error) { - +func getDefsByPackNameFromOvalDB(r *models.ScanResult) (relatedDefs ovalResult, err error) { ovallog.Initialize(config.Conf.LogDir) path := config.Conf.OvalDBURL if config.Conf.OvalDBType == "sqlite3" { @@ -273,48 +224,96 @@ func getDefsByPackNameFromOvalDB(family, osRelease string, util.Log.Debugf("Open oval-dictionary db (%s): %s", config.Conf.OvalDBType, path) var ovaldb db.DB - if ovaldb, err = db.NewDB( - family, - config.Conf.OvalDBType, - path, - config.Conf.DebugSQL, - ); err != nil { + if ovaldb, err = db.NewDB(r.Family, config.Conf.OvalDBType, + path, config.Conf.DebugSQL); err != nil { return } defer ovaldb.CloseDB() - for _, installedPack := range installedPacks { - definitions, err := ovaldb.GetByPackName(osRelease, installedPack.Name) + + requests := []request{} + for _, pack := range r.Packages { + requests = append(requests, request{ + packName: pack.Name, + versionRelease: pack.FormatVer(), + NewVersionRelease: pack.FormatNewVer(), + isSrcPack: false, + }) + } + for _, pack := range r.SrcPackages { + requests = append(requests, request{ + packName: pack.Name, + binaryPackNames: pack.BinaryNames, + versionRelease: pack.Version, + isSrcPack: true, + }) + } + + for _, req := range requests { + definitions, err := ovaldb.GetByPackName(r.Release, req.packName) if err != nil { - return relatedDefs, fmt.Errorf("Failed to get %s OVAL info by package name: %v", family, err) + return relatedDefs, fmt.Errorf("Failed to get %s OVAL info by package name: %v", r.Family, err) } for _, def := range definitions { - for _, ovalPack := range def.AffectedPacks { - if installedPack.Name != ovalPack.Name { - continue - } + affected, notFixedYet := isOvalDefAffected(def, r.Family, req) + if !affected { + continue + } - if ovalPack.NotFixedYet { - relatedDefs.upsert(def, installedPack.Name) - continue - } - - less, err := lessThan(family, installedPack, ovalPack) - if err != nil { - util.Log.Debugf("Failed to parse versions: %s", err) - util.Log.Debugf("%#v\n%#v", installedPack, ovalPack) - } else if less { - relatedDefs.upsert(def, installedPack.Name) + if req.isSrcPack { + for _, n := range req.binaryPackNames { + relatedDefs.upsert(def, n, false) } + } else { + relatedDefs.upsert(def, req.packName, notFixedYet) } } } return } -func lessThan(family string, packA models.Package, packB ovalmodels.Package) (bool, error) { +func isOvalDefAffected(def ovalmodels.Definition, family string, req request) (affected, notFixedYet bool) { + for _, ovalPack := range def.AffectedPacks { + if req.packName != ovalPack.Name { + continue + } + + if ovalPack.NotFixedYet { + return true, true + } + + less, err := lessThan(family, req.versionRelease, ovalPack) + if err != nil { + util.Log.Debugf("Failed to parse versions: %s, Ver: %#v, OVAL: %#v, DefID: %s", + err, req.versionRelease, ovalPack, def.DefinitionID) + return false, false + } + + if less { + if req.isSrcPack { + // Unable to judge whether fixed or not fixed of src package(Ubuntu, Debian) + return true, false + } + if req.NewVersionRelease == "" { + return true, true + } + + // compare version: newVer vs oval + less, err := lessThan(family, req.NewVersionRelease, ovalPack) + if err != nil { + util.Log.Debugf("Failed to parse versions: %s, NewVer: %#v, OVAL: %#v, DefID: %s", + err, req.NewVersionRelease, ovalPack, def.DefinitionID) + return false, false + } + return true, less + } + } + return false, false +} + +func lessThan(family, versionRelease string, packB ovalmodels.Package) (bool, error) { switch family { case config.Debian, config.Ubuntu: - vera, err := debver.NewVersion(packA.Version) + vera, err := debver.NewVersion(versionRelease) if err != nil { return false, err } @@ -324,7 +323,7 @@ func lessThan(family string, packA models.Package, packB ovalmodels.Package) (bo } return vera.LessThan(verb), nil case config.RedHat, config.CentOS, config.Oracle, config.SUSEEnterpriseServer: - vera := rpmver.NewVersion(fmt.Sprintf("%s-%s", packA.Version, packA.Release)) + vera := rpmver.NewVersion(versionRelease) verb := rpmver.NewVersion(packB.Version) return vera.LessThan(verb), nil default: diff --git a/oval/util_test.go b/oval/util_test.go index 241b16af..1631490c 100644 --- a/oval/util_test.go +++ b/oval/util_test.go @@ -11,11 +11,12 @@ import ( func TestUpsert(t *testing.T) { var tests = []struct { - res ovalResult - def ovalmodels.Definition - packName string - upserted bool - out ovalResult + res ovalResult + def ovalmodels.Definition + packName string + notFixedYet bool + upserted bool + out ovalResult }{ //insert { @@ -23,8 +24,9 @@ func TestUpsert(t *testing.T) { def: ovalmodels.Definition{ DefinitionID: "1111", }, - packName: "pack1", - upserted: false, + packName: "pack1", + notFixedYet: true, + upserted: false, out: ovalResult{ []defPacks{ { @@ -63,8 +65,9 @@ func TestUpsert(t *testing.T) { def: ovalmodels.Definition{ DefinitionID: "1111", }, - packName: "pack2", - upserted: true, + packName: "pack2", + notFixedYet: false, + upserted: true, out: ovalResult{ []defPacks{ { @@ -73,7 +76,7 @@ func TestUpsert(t *testing.T) { }, actuallyAffectedPackNames: map[string]bool{ "pack1": true, - "pack2": true, + "pack2": false, }, }, { @@ -89,7 +92,7 @@ func TestUpsert(t *testing.T) { }, } for i, tt := range tests { - upserted := tt.res.upsert(tt.def, tt.packName) + upserted := tt.res.upsert(tt.def, tt.packName, tt.notFixedYet) if tt.upserted != upserted { t.Errorf("[%d]\nexpected: %t\n actual: %t\n", i, tt.upserted, upserted) } @@ -127,90 +130,6 @@ func TestDefpacksToPackStatuses(t *testing.T) { }, }, }, - actuallyAffectedPackNames: map[string]bool{ - "a": true, - "b": true, - }, - }, - }, - out: models.PackageStatuses{ - { - Name: "a", - NotFixedYet: true, - }, - { - Name: "b", - NotFixedYet: false, - }, - }, - }, - - // RedHat, Amazon, Debian - { - in: in{ - family: "redhat", - packs: models.Packages{}, - dp: defPacks{ - def: ovalmodels.Definition{ - AffectedPacks: []ovalmodels.Package{ - { - Name: "a", - }, - { - Name: "b", - }, - }, - }, - actuallyAffectedPackNames: map[string]bool{ - "a": true, - "b": true, - }, - }, - }, - out: models.PackageStatuses{ - { - Name: "a", - NotFixedYet: false, - }, - { - Name: "b", - NotFixedYet: false, - }, - }, - }, - - // CentOS - { - in: in{ - family: "centos", - packs: models.Packages{ - "a": {Version: "1.0.0"}, - "b": { - Version: "1.0.0", - NewVersion: "2.0.0", - }, - "c": { - Version: "1.0.0", - NewVersion: "1.5.0", - }, - }, - dp: defPacks{ - def: ovalmodels.Definition{ - AffectedPacks: []ovalmodels.Package{ - { - Name: "a", - Version: "1.0.1", - }, - { - Name: "b", - Version: "1.5.0", - }, - { - Name: "c", - Version: "2.0.0", - }, - }, - }, actuallyAffectedPackNames: map[string]bool{ "a": true, "b": true, @@ -225,7 +144,7 @@ func TestDefpacksToPackStatuses(t *testing.T) { }, { Name: "b", - NotFixedYet: false, + NotFixedYet: true, }, { Name: "c", @@ -244,3 +163,171 @@ func TestDefpacksToPackStatuses(t *testing.T) { } } } + +func TestIsOvalDefAffected(t *testing.T) { + type in struct { + def ovalmodels.Definition + family string + req request + } + var tests = []struct { + in in + affected bool + notFixedYet bool + }{ + // 0. Ubuntu ovalpack.NotFixedYet == true + { + in: in{ + family: "ubuntu", + def: ovalmodels.Definition{ + AffectedPacks: []ovalmodels.Package{ + { + Name: "a", + NotFixedYet: true, + }, + { + Name: "b", + NotFixedYet: true, + }, + }, + }, + req: request{ + packName: "b", + }, + }, + affected: true, + notFixedYet: true, + }, + // 1. Ubuntu + // ovalpack.NotFixedYet == false + // req.isSrcPack == true + // Version comparison + // oval vs installed + { + in: in{ + family: "ubuntu", + def: ovalmodels.Definition{ + AffectedPacks: []ovalmodels.Package{ + { + Name: "a", + NotFixedYet: false, + }, + { + Name: "b", + NotFixedYet: false, + Version: "1.0.0-1", + }, + }, + }, + req: request{ + packName: "b", + isSrcPack: true, + versionRelease: "1.0.0-0", + }, + }, + affected: true, + notFixedYet: false, + }, + // 2. Ubuntu + // ovalpack.NotFixedYet == false + // Version comparison not hit + // oval vs installed + { + in: in{ + family: "ubuntu", + def: ovalmodels.Definition{ + AffectedPacks: []ovalmodels.Package{ + { + Name: "a", + NotFixedYet: false, + }, + { + Name: "b", + NotFixedYet: false, + Version: "1.0.0-1", + }, + }, + }, + req: request{ + packName: "b", + versionRelease: "1.0.0-2", + }, + }, + affected: false, + notFixedYet: false, + }, + // 3. Ubuntu + // ovalpack.NotFixedYet == false + // req.isSrcPack == false + // Version comparison + // oval vs NewVersion + // oval.version < installed.newVersion + { + in: in{ + family: "ubuntu", + def: ovalmodels.Definition{ + AffectedPacks: []ovalmodels.Package{ + { + Name: "a", + NotFixedYet: false, + }, + { + Name: "b", + NotFixedYet: false, + Version: "1.0.0-3", + }, + }, + }, + req: request{ + packName: "b", + isSrcPack: false, + versionRelease: "1.0.0-0", + NewVersionRelease: "1.0.0-2", + }, + }, + affected: true, + notFixedYet: true, + }, + // 4. Ubuntu + // ovalpack.NotFixedYet == false + // req.isSrcPack == false + // Version comparison + // oval vs NewVersion + // oval.version < installed.newVersion + { + in: in{ + family: "ubuntu", + def: ovalmodels.Definition{ + AffectedPacks: []ovalmodels.Package{ + { + Name: "a", + NotFixedYet: false, + }, + { + Name: "b", + NotFixedYet: false, + Version: "1.0.0-2", + }, + }, + }, + req: request{ + packName: "b", + isSrcPack: false, + versionRelease: "1.0.0-0", + NewVersionRelease: "1.0.0-3", + }, + }, + affected: true, + notFixedYet: false, + }, + } + for i, tt := range tests { + affected, notFixedYet := isOvalDefAffected(tt.in.def, tt.in.family, tt.in.req) + if tt.affected != affected { + t.Errorf("[%d] affected\nexpected: %v\n actual: %v\n", i, tt.affected, affected) + } + if tt.notFixedYet != notFixedYet { + t.Errorf("[%d] notfixedyet\nexpected: %v\n actual: %v\n", i, tt.notFixedYet, notFixedYet) + } + } +} diff --git a/scan/base.go b/scan/base.go index ba02a346..8aa42370 100644 --- a/scan/base.go +++ b/scan/base.go @@ -313,6 +313,7 @@ func (l *base) convertToModel() models.ScanResult { ScannedCves: l.VulnInfos, RunningKernel: l.Kernel, Packages: l.Packages, + SrcPackages: l.SrcPackages, Optional: l.ServerInfo.Optional, Errors: errs, } diff --git a/scan/debian.go b/scan/debian.go index b791d729..739f10c5 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -176,7 +176,6 @@ func (o *debian) checkDependencies() error { } for _, name := range packNames { - //TODO --show-format cmd := "dpkg-query -W " + name if r := o.exec(cmd, noSudo); !r.isSuccess() { msg := fmt.Sprintf("%s is not installed", name) @@ -206,12 +205,13 @@ func (o *debian) scanPackages() error { RebootRequired: rebootRequired, } - installed, updatable, err := o.scanInstalledPackages() + installed, updatable, srcPacks, err := o.scanInstalledPackages() if err != nil { o.log.Errorf("Failed to scan installed packages: %s", err) return err } o.Packages = installed + o.SrcPackages = srcPacks if config.Conf.Deep || o.Distro.Family == config.Raspbian { unsecures, err := o.scanUnsecurePackages(updatable) @@ -238,20 +238,26 @@ func (o *debian) rebootRequired() (bool, error) { } } -func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, error) { - installed, updatable := models.Packages{}, models.Packages{} - r := o.exec(`dpkg-query -W -f='${binary:Package}\t${db:Status-Abbrev}\t${Version}\n'`, noSudo) +func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, models.SrcPackages, error) { + installed, updatable, srcPacks := models.Packages{}, models.Packages{}, models.SrcPackages{} + r := o.exec(`dpkg-query -W -f='${binary:Package},${db:Status-Abbrev},${Version},${Source},${source:Version}\n'`, noSudo) if !r.isSuccess() { - return nil, nil, fmt.Errorf("Failed to SSH: %s", r) + return nil, nil, nil, fmt.Errorf("Failed to SSH: %s", r) } - // e.g. - // curl ii 7.47.0-1ubuntu2.2 - // openssh-server rc 1:7.2p2-4ubuntu2.2 + // e.g. + // curl,ii ,7.38.0-4+deb8u2,,7.38.0-4+deb8u2 + // openssh-server,ii ,1:6.7p1-5+deb8u3,openssh,1:6.7p1-5+deb8u3 + // tar,ii ,1.27.1-2+b1,tar (1.27.1-2),1.27.1-2 lines := strings.Split(r.Stdout, "\n") for _, line := range lines { if trimmed := strings.TrimSpace(line); len(trimmed) != 0 { - name, status, version, err := o.parseScannedPackagesLine(trimmed) + name, status, version, srcName, srcVersion, err := o.parseScannedPackagesLine(trimmed) + if err != nil { + return nil, nil, nil, fmt.Errorf( + "Debian: Failed to parse package line: %s", line) + } + packageStatus := status[1] // Package status: // n = Not-installed @@ -266,20 +272,38 @@ func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, erro o.log.Debugf("%s package status is '%c', ignoring", name, packageStatus) continue } - if err != nil { - return nil, nil, fmt.Errorf( - "Debian: Failed to parse package line: %s", line) - } installed[name] = models.Package{ Name: name, Version: version, } + + if srcName != "" && srcName != name { + if pack, ok := srcPacks[srcName]; ok { + pack.AddBinaryName(name) + srcPacks[srcName] = pack + } else { + srcPacks[srcName] = models.SrcPackage{ + Name: srcName, + Version: srcVersion, + BinaryNames: []string{name}, + } + } + } } } + // Remove "linux" + // kernel-related packages are showed "linux" as source package name + // If "linux" is left, oval detection will cause trouble, so delete. + delete(srcPacks, "linux") + // Remove duplicate + for name := range installed { + delete(srcPacks, name) + } + updatableNames, err := o.getUpdatablePackNames() if err != nil { - return nil, nil, err + return nil, nil, nil, err } for _, name := range updatableNames { for _, pack := range installed { @@ -293,29 +317,30 @@ func (o *debian) scanInstalledPackages() (models.Packages, models.Packages, erro // Fill the candidate versions of upgradable packages err = o.fillCandidateVersion(updatable) if err != nil { - return nil, nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err) + return nil, nil, nil, fmt.Errorf("Failed to fill candidate versions. err: %s", err) } installed.MergeNewVersion(updatable) - return installed, updatable, nil + return installed, updatable, srcPacks, nil } -var packageLinePattern = regexp.MustCompile(`^([^\t']+)\t(.+)\t(.+)$`) - -func (o *debian) parseScannedPackagesLine(line string) (name, status, version string, err error) { - result := packageLinePattern.FindStringSubmatch(line) - if len(result) == 4 { +func (o *debian) parseScannedPackagesLine(line string) (name, status, version, srcName, srcVersion string, err error) { + ss := strings.Split(line, ",") + if len(ss) == 5 { // remove :amd64, i386... - name = result[1] + name = ss[0] if i := strings.IndexRune(name, ':'); i >= 0 { name = name[:i] } - status = result[2] - version = result[3] + status = ss[1] + version = ss[2] + // remove version. ex: tar (1.27.1-2) + srcName = strings.Split(ss[3], " ")[0] + srcVersion = ss[4] return } - return "", "", "", fmt.Errorf("Unknown format: %s", line) + return "", "", "", "", "", fmt.Errorf("Unknown format: %s", line) } func (o *debian) aptGetUpdate() error { diff --git a/scan/debian_test.go b/scan/debian_test.go index f7cef683..02384125 100644 --- a/scan/debian_test.go +++ b/scan/debian_test.go @@ -29,39 +29,6 @@ import ( "github.com/sirupsen/logrus" ) -func TestParseScannedPackagesLineDebian(t *testing.T) { - - var packagetests = []struct { - in string - name string - status string - version string - }{ - {"base-passwd ii 3.5.33", "base-passwd", "ii ", "3.5.33"}, - {"bzip2 ii 1.0.6-5", "bzip2", "ii ", "1.0.6-5"}, - {"adduser ii 3.113+nmu3ubuntu3", "adduser", "ii ", "3.113+nmu3ubuntu3"}, - {"bash ii 4.3-7ubuntu1.5", "bash", "ii ", "4.3-7ubuntu1.5"}, - {"bsdutils ii 1:2.20.1-5.1ubuntu20.4", "bsdutils", "ii ", "1:2.20.1-5.1ubuntu20.4"}, - {"ca-certificates ii 20141019ubuntu0.14.04.1", "ca-certificates", "ii ", "20141019ubuntu0.14.04.1"}, - {"apt rc 1.0.1ubuntu2.8", "apt", "rc ", "1.0.1ubuntu2.8"}, - } - - d := newDebian(config.ServerInfo{}) - for _, tt := range packagetests { - n, s, v, _ := d.parseScannedPackagesLine(tt.in) - if n != tt.name { - t.Errorf("name: expected %s, actual %s", tt.name, n) - } - if s != tt.status { - t.Errorf("status: expected %s, actual %s", tt.status, s) - } - if v != tt.version { - t.Errorf("version: expected %s, actual %s", tt.version, v) - } - } - -} - func TestGetCveIDsFromChangelog(t *testing.T) { var tests = []struct { diff --git a/scan/redhat.go b/scan/redhat.go index 6ebb02e8..75dfefc9 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -385,7 +385,7 @@ func (o *redhat) parseUpdatablePacksLine(line string) (models.Package, error) { ver = fmt.Sprintf("%s:%s", epoch, fields[2]) } - repos := strings.Join(fields[4:len(fields)], " ") + repos := strings.Join(fields[4:], " ") p := models.Package{ Name: fields[0], @@ -816,7 +816,7 @@ func (o *redhat) parseYumUpdateinfo(stdout string) (result []distroAdvisoryCveID inDesctiption, inCves = true, false ss := strings.Split(line, " : ") advisory.Description += fmt.Sprintf("%s\n", - strings.Join(ss[1:len(ss)], " : ")) + strings.Join(ss[1:], " : ")) continue } @@ -830,7 +830,7 @@ func (o *redhat) parseYumUpdateinfo(stdout string) (result []distroAdvisoryCveID if inDesctiption { if ss := strings.Split(line, ": "); 1 < len(ss) { advisory.Description += fmt.Sprintf("%s\n", - strings.Join(ss[1:len(ss)], ": ")) + strings.Join(ss[1:], ": ")) } continue } @@ -838,7 +838,7 @@ func (o *redhat) parseYumUpdateinfo(stdout string) (result []distroAdvisoryCveID if found := o.isCvesHeaderLine(line); found { inCves = true ss := strings.Split(line, "CVEs : ") - line = strings.Join(ss[1:len(ss)], " ") + line = strings.Join(ss[1:], " ") cveIDs := o.parseYumUpdateinfoLineToGetCveIDs(line) for _, cveID := range cveIDs { cveIDsSetInThisSection[cveID] = true diff --git a/scan/serverapi.go b/scan/serverapi.go index a6d0b52b..7d1cc507 100644 --- a/scan/serverapi.go +++ b/scan/serverapi.go @@ -61,6 +61,9 @@ type osPackages struct { // installed packages Packages models.Packages + // installed source packages (Debian based only) + SrcPackages models.SrcPackages + // unsecure packages VulnInfos models.VulnInfos