From 5c51d83573a81c174fdccfeee88a0b8fd1400373 Mon Sep 17 00:00:00 2001 From: kota kanbe Date: Sun, 13 Aug 2017 17:18:01 +0900 Subject: [PATCH] Refactoring --- oval/debian.go | 78 ++++++----------- oval/oval.go | 153 +++++--------------------------- oval/redhat.go | 76 ++++++---------- oval/util.go | 233 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 305 insertions(+), 235 deletions(-) create mode 100644 oval/util.go diff --git a/oval/debian.go b/oval/debian.go index ad27dce9..e7e05523 100644 --- a/oval/debian.go +++ b/oval/debian.go @@ -1,26 +1,36 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ package oval import ( - "fmt" - "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" - ver "github.com/knqyf263/go-deb-version" - db "github.com/kotakanbe/goval-dictionary/db" - ovallog "github.com/kotakanbe/goval-dictionary/log" ovalmodels "github.com/kotakanbe/goval-dictionary/models" ) // DebianBase is the base struct of Debian and Ubuntu type DebianBase struct { Base - family string } // fillFromOvalDB returns scan result after updating CVE info by OVAL func (o DebianBase) fillFromOvalDB(r *models.ScanResult) error { - defs, err := o.getDefsByPackNameFromOvalDB(r.Release, r.Packages) + defs, err := getDefsByPackNameFromOvalDB(o.family, r.Release, r.Packages) if err != nil { return err } @@ -30,51 +40,9 @@ func (o DebianBase) fillFromOvalDB(r *models.ScanResult) error { return nil } -func (o DebianBase) getDefsByPackNameFromOvalDB(osRelease string, - packs models.Packages) (relatedDefs []ovalmodels.Definition, err error) { - - ovallog.Initialize(config.Conf.LogDir) - path := config.Conf.OvalDBURL - if config.Conf.OvalDBType == "sqlite3" { - path = config.Conf.OvalDBPath - } - util.Log.Debugf("Open oval-dictionary db (%s): %s", config.Conf.OvalDBType, path) - - var ovaldb db.DB - if ovaldb, err = db.NewDB( - o.family, - config.Conf.OvalDBType, - path, - config.Conf.DebugSQL, - ); err != nil { - return - } - defer ovaldb.CloseDB() - - for _, pack := range packs { - definitions, err := ovaldb.GetByPackName(osRelease, pack.Name) - if err != nil { - return nil, fmt.Errorf("Failed to get %s OVAL info by package name: %v", o.family, err) - } - for _, def := range definitions { - current, _ := ver.NewVersion(pack.Version) - for _, p := range def.AffectedPacks { - if pack.Name != p.Name { - continue - } - affected, _ := ver.NewVersion(p.Version) - if current.LessThan(affected) { - relatedDefs = append(relatedDefs, def) - } - } - } - } - return -} - func (o DebianBase) update(r *models.ScanResult, definition *ovalmodels.Definition) { ovalContent := *o.convertToModel(definition) - ovalContent.Type = models.NewCveContentType(r.Family) + ovalContent.Type = models.NewCveContentType(o.family) vinfo, ok := r.ScannedCves[definition.Debian.CveID] if !ok { util.Log.Debugf("%s is newly detected by OVAL", definition.Debian.CveID) @@ -86,7 +54,7 @@ func (o DebianBase) update(r *models.ScanResult, definition *ovalmodels.Definiti } } else { cveContents := vinfo.CveContents - ctype := models.NewCveContentType(r.Family) + ctype := models.NewCveContentType(o.family) if _, ok := vinfo.CveContents[ctype]; ok { util.Log.Debugf("%s will be updated by OVAL", definition.Debian.CveID) } else { @@ -130,7 +98,9 @@ type Debian struct { func NewDebian() Debian { return Debian{ DebianBase{ - family: config.Debian, + Base{ + family: config.Debian, + }, }, } } @@ -170,7 +140,9 @@ type Ubuntu struct { func NewUbuntu() Ubuntu { return Ubuntu{ DebianBase{ - family: config.Ubuntu, + Base{ + family: config.Ubuntu, + }, }, } } diff --git a/oval/oval.go b/oval/oval.go index a4f15965..591c8853 100644 --- a/oval/oval.go +++ b/oval/oval.go @@ -1,3 +1,20 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + package oval import ( @@ -7,14 +24,11 @@ import ( "strings" "time" - "github.com/cenkalti/backoff" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" - ver "github.com/knqyf263/go-deb-version" "github.com/kotakanbe/goval-dictionary/db" ovallog "github.com/kotakanbe/goval-dictionary/log" - ovalmodels "github.com/kotakanbe/goval-dictionary/models" "github.com/parnurzeal/gorequest" ) @@ -29,7 +43,9 @@ type Client interface { } // Base is a base struct -type Base struct{} +type Base struct { + family string +} // CheckHTTPHealth do health check func (b Base) CheckHTTPHealth() error { @@ -135,132 +151,3 @@ func (b Base) isFetchViaHTTP() bool { } return false } - -type request struct { - pack models.Package -} - -type response struct { - pack *models.Package - defs []ovalmodels.Definition -} - -// getDefsByPackNameViaHTTP fetches OVAL information via HTTP -func getDefsByPackNameViaHTTP(r *models.ScanResult) ( - relatedDefs []ovalmodels.Definition, err error) { - - reqChan := make(chan request, len(r.Packages)) - resChan := make(chan response, len(r.Packages)) - errChan := make(chan error, len(r.Packages)) - defer close(reqChan) - defer close(resChan) - defer close(errChan) - - go func() { - for _, pack := range r.Packages { - reqChan <- request{ - pack: pack, - } - } - }() - - concurrency := 10 - tasks := util.GenWorkers(concurrency) - for range r.Packages { - tasks <- func() { - select { - case req := <-reqChan: - url, err := util.URLPathJoin( - config.Conf.OvalDBURL, - "packs", - r.Family, - r.Release, - req.pack.Name, - ) - if err != nil { - errChan <- err - } else { - util.Log.Debugf("HTTP Request to %s", url) - httpGet(url, &req.pack, resChan, errChan) - } - } - } - } - - timeout := time.After(2 * 60 * time.Second) - var errs []error - for range r.Packages { - select { - case res := <-resChan: - current, _ := ver.NewVersion(fmt.Sprintf("%s-%s", - res.pack.Version, res.pack.Release)) - for _, def := range res.defs { - for _, p := range def.AffectedPacks { - affected, _ := ver.NewVersion(p.Version) - if res.pack.Name != p.Name || !current.LessThan(affected) { - continue - } - relatedDefs = append(relatedDefs, def) - } - } - case err := <-errChan: - errs = append(errs, err) - case <-timeout: - return nil, fmt.Errorf("Timeout Fetching OVAL") - } - } - if len(errs) != 0 { - return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs) - } - return -} - -func httpGet(url string, pack *models.Package, resChan chan<- response, errChan chan<- error) { - var body string - var errs []error - var resp *http.Response - count, retryMax := 0, 3 - f := func() (err error) { - // resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End() - resp, body, errs = gorequest.New().Get(url).End() - if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { - count++ - if count == retryMax { - return nil - } - return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", - errs, url, resp) - } - return nil - } - notify := func(err error, t time.Duration) { - util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err) - } - err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) - if err != nil { - errChan <- fmt.Errorf("HTTP Error %s", err) - return - } - if count == retryMax { - errChan <- fmt.Errorf("HRetry count exceeded") - return - } - - defs := []ovalmodels.Definition{} - if err := json.Unmarshal([]byte(body), &defs); err != nil { - errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", - body, err) - return - } - resChan <- response{ - pack: pack, - defs: defs, - } -} - -func getPackages(r *models.ScanResult, d *ovalmodels.Definition) (names []string) { - for _, affectedPack := range d.AffectedPacks { - names = append(names, affectedPack.Name) - } - return -} diff --git a/oval/redhat.go b/oval/redhat.go index b1a76130..2541c3cb 100644 --- a/oval/redhat.go +++ b/oval/redhat.go @@ -1,3 +1,20 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + package oval import ( @@ -8,16 +25,12 @@ import ( "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" - ver "github.com/knqyf263/go-rpm-version" - db "github.com/kotakanbe/goval-dictionary/db" - ovallog "github.com/kotakanbe/goval-dictionary/log" ovalmodels "github.com/kotakanbe/goval-dictionary/models" ) // RedHatBase is the base struct for RedHat and CentOS type RedHatBase struct { Base - family string } // FillWithOval returns scan result after updating CVE info by OVAL @@ -54,7 +67,7 @@ func (o RedHatBase) FillWithOval(r *models.ScanResult) error { // fillFromOvalDB returns scan result after updating CVE info by OVAL func (o RedHatBase) fillFromOvalDB(r *models.ScanResult) error { - defs, err := o.getDefsByPackNameFromOvalDB(r.Release, r.Packages) + defs, err := getDefsByPackNameFromOvalDB(o.family, r.Release, r.Packages) if err != nil { return err } @@ -64,47 +77,6 @@ func (o RedHatBase) fillFromOvalDB(r *models.ScanResult) error { return nil } -func (o RedHatBase) getDefsByPackNameFromOvalDB(osRelease string, - packs models.Packages) (relatedDefs []ovalmodels.Definition, err error) { - - ovallog.Initialize(config.Conf.LogDir) - path := config.Conf.OvalDBURL - if config.Conf.OvalDBType == "sqlite3" { - path = config.Conf.OvalDBPath - } - util.Log.Debugf("Open oval-dictionary db (%s): %s", config.Conf.OvalDBType, path) - - var ovaldb db.DB - if ovaldb, err = db.NewDB( - o.family, - config.Conf.OvalDBType, - path, - config.Conf.DebugSQL, - ); err != nil { - return - } - defer ovaldb.CloseDB() - for _, pack := range packs { - definitions, err := ovaldb.GetByPackName(osRelease, pack.Name) - if err != nil { - return nil, fmt.Errorf("Failed to get %s OVAL info by package name: %v", o.family, err) - } - for _, def := range definitions { - current := ver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release)) - for _, p := range def.AffectedPacks { - if pack.Name != p.Name { - continue - } - affected := ver.NewVersion(p.Version) - if current.LessThan(affected) { - relatedDefs = append(relatedDefs, def) - } - } - } - } - return -} - func (o RedHatBase) update(r *models.ScanResult, definition *ovalmodels.Definition) { ctype := models.NewCveContentType(o.family) for _, cve := range definition.Advisory.Cves { @@ -210,7 +182,9 @@ type RedHat struct { func NewRedhat() RedHat { return RedHat{ RedHatBase{ - family: config.RedHat, + Base{ + family: config.RedHat, + }, }, } } @@ -224,7 +198,9 @@ type CentOS struct { func NewCentOS() CentOS { return CentOS{ RedHatBase{ - family: config.CentOS, + Base{ + family: config.CentOS, + }, }, } } @@ -238,7 +214,9 @@ type Oracle struct { func NewOracle() Oracle { return Oracle{ RedHatBase{ - family: config.Oracle, + Base{ + family: config.Oracle, + }, }, } } diff --git a/oval/util.go b/oval/util.go new file mode 100644 index 00000000..c84cade1 --- /dev/null +++ b/oval/util.go @@ -0,0 +1,233 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package oval + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/cenkalti/backoff" + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + debver "github.com/knqyf263/go-deb-version" + rpmver "github.com/knqyf263/go-rpm-version" + "github.com/kotakanbe/goval-dictionary/db" + ovallog "github.com/kotakanbe/goval-dictionary/log" + ovalmodels "github.com/kotakanbe/goval-dictionary/models" + "github.com/parnurzeal/gorequest" +) + +type request struct { + pack models.Package +} + +type response struct { + pack *models.Package + defs []ovalmodels.Definition +} + +// getDefsByPackNameViaHTTP fetches OVAL information via HTTP +func getDefsByPackNameViaHTTP(r *models.ScanResult) ( + relatedDefs []ovalmodels.Definition, err error) { + + reqChan := make(chan request, len(r.Packages)) + resChan := make(chan response, len(r.Packages)) + errChan := make(chan error, len(r.Packages)) + defer close(reqChan) + defer close(resChan) + defer close(errChan) + + go func() { + for _, pack := range r.Packages { + reqChan <- request{ + pack: pack, + } + } + }() + + concurrency := 10 + tasks := util.GenWorkers(concurrency) + for range r.Packages { + tasks <- func() { + select { + case req := <-reqChan: + url, err := util.URLPathJoin( + config.Conf.OvalDBURL, + "packs", + r.Family, + r.Release, + req.pack.Name, + ) + if err != nil { + errChan <- err + } else { + util.Log.Debugf("HTTP Request to %s", url) + httpGet(url, &req.pack, resChan, errChan) + } + } + } + } + + timeout := time.After(2 * 60 * time.Second) + var errs []error + for range r.Packages { + select { + case res := <-resChan: + for _, def := range res.defs { + for _, p := range def.AffectedPacks { + if res.pack.Name != p.Name { + continue + } + if less, err := lessThan(r.Family, *res.pack, p); err != nil { + if !p.NotFixedYet { + util.Log.Debugf("Failed to parse versions: %s", err) + util.Log.Debugf("%#v\n%#v", *res.pack, p) + } + } else if less { + relatedDefs = append(relatedDefs, def) + } + } + } + case err := <-errChan: + errs = append(errs, err) + case <-timeout: + return nil, fmt.Errorf("Timeout Fetching OVAL") + } + } + if len(errs) != 0 { + return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs) + } + return +} + +func httpGet(url string, pack *models.Package, resChan chan<- response, errChan chan<- error) { + var body string + var errs []error + var resp *http.Response + count, retryMax := 0, 3 + f := func() (err error) { + // resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End() + resp, body, errs = gorequest.New().Get(url).End() + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { + count++ + if count == retryMax { + return nil + } + return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v", + errs, url, resp) + } + return nil + } + notify := func(err error, t time.Duration) { + util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err) + } + err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify) + if err != nil { + errChan <- fmt.Errorf("HTTP Error %s", err) + return + } + if count == retryMax { + errChan <- fmt.Errorf("HRetry count exceeded") + return + } + + defs := []ovalmodels.Definition{} + if err := json.Unmarshal([]byte(body), &defs); err != nil { + errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s", + body, err) + return + } + resChan <- response{ + pack: pack, + defs: defs, + } +} + +func getPackages(r *models.ScanResult, d *ovalmodels.Definition) (names []string) { + for _, affectedPack := range d.AffectedPacks { + names = append(names, affectedPack.Name) + } + return +} + +func getDefsByPackNameFromOvalDB(family, osRelease string, + packs models.Packages) (relatedDefs []ovalmodels.Definition, err error) { + + ovallog.Initialize(config.Conf.LogDir) + path := config.Conf.OvalDBURL + if config.Conf.OvalDBType == "sqlite3" { + path = config.Conf.OvalDBPath + } + util.Log.Debugf("Open oval-dictionary db (%s): %s", config.Conf.OvalDBType, path) + + var ovaldb db.DB + if ovaldb, err = db.NewDB( + family, + config.Conf.OvalDBType, + path, + config.Conf.DebugSQL, + ); err != nil { + return + } + defer ovaldb.CloseDB() + for _, pack := range packs { + definitions, err := ovaldb.GetByPackName(osRelease, pack.Name) + if err != nil { + return nil, fmt.Errorf("Failed to get %s OVAL info by package name: %v", family, err) + } + for _, def := range definitions { + for _, p := range def.AffectedPacks { + if pack.Name != p.Name { + continue + } + if less, err := lessThan(family, pack, p); err != nil { + if !p.NotFixedYet { + util.Log.Debugf("Failed to parse versions: %s", err) + util.Log.Debugf("%#v\n%#v", pack, p) + } + } else if less { + relatedDefs = append(relatedDefs, def) + } + } + } + } + return +} + +func lessThan(family string, packA models.Package, packB ovalmodels.Package) (bool, error) { + switch family { + case config.Debian, config.Ubuntu: + vera, err := debver.NewVersion(packA.Version) + if err != nil { + return false, err + } + verb, err := debver.NewVersion(packB.Version) + if err != nil { + return false, err + } + return vera.LessThan(verb), nil + case config.RedHat, config.CentOS, config.Oracle: + vera := rpmver.NewVersion(fmt.Sprintf("%s-%s", packA.Version, packA.Release)) + verb := rpmver.NewVersion(packB.Version) + return vera.LessThan(verb), nil + } + return false, fmt.Errorf("Package version comparison not supported: %s", family) +}