Stage/vuls
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring

Browse Source
This commit is contained in:
kota kanbe
2017-08-13 17:18:01 +09:00
parent 47b3b3848b
commit 5c51d83573
4 changed files with 305 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
oval/debian.go
View File

@@ -1,26 +1,36 @@
/* Vuls - Vulnerability Scanner
Copyright (C) 2016 Future Architect, Inc. Japan.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package oval
import (
"fmt"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
ver "github.com/knqyf263/go-deb-version"
db "github.com/kotakanbe/goval-dictionary/db"
ovallog "github.com/kotakanbe/goval-dictionary/log"
ovalmodels "github.com/kotakanbe/goval-dictionary/models"
)
// DebianBase is the base struct of Debian and Ubuntu
type DebianBase struct {
Base
family string
}
// fillFromOvalDB returns scan result after updating CVE info by OVAL
func (o DebianBase) fillFromOvalDB(r *models.ScanResult) error {
defs, err := o.getDefsByPackNameFromOvalDB(r.Release, r.Packages)
defs, err := getDefsByPackNameFromOvalDB(o.family, r.Release, r.Packages)
if err != nil {
return err
}
@@ -30,51 +40,9 @@ func (o DebianBase) fillFromOvalDB(r *models.ScanResult) error {
return nil
}
func (o DebianBase) getDefsByPackNameFromOvalDB(osRelease string,
packs models.Packages) (relatedDefs []ovalmodels.Definition, err error) {
ovallog.Initialize(config.Conf.LogDir)
path := config.Conf.OvalDBURL
if config.Conf.OvalDBType == "sqlite3" {
path = config.Conf.OvalDBPath
}
util.Log.Debugf("Open oval-dictionary db (%s): %s", config.Conf.OvalDBType, path)
var ovaldb db.DB
if ovaldb, err = db.NewDB(
o.family,
config.Conf.OvalDBType,
path,
config.Conf.DebugSQL,
); err != nil {
return
}
defer ovaldb.CloseDB()
for _, pack := range packs {
definitions, err := ovaldb.GetByPackName(osRelease, pack.Name)
if err != nil {
return nil, fmt.Errorf("Failed to get %s OVAL info by package name: %v", o.family, err)
}
for _, def := range definitions {
current, _ := ver.NewVersion(pack.Version)
for _, p := range def.AffectedPacks {
if pack.Name != p.Name {
continue
}
affected, _ := ver.NewVersion(p.Version)
if current.LessThan(affected) {
relatedDefs = append(relatedDefs, def)
}
}
}
}
return
}
func (o DebianBase) update(r *models.ScanResult, definition *ovalmodels.Definition) {
ovalContent := *o.convertToModel(definition)
ovalContent.Type = models.NewCveContentType(r.Family)
ovalContent.Type = models.NewCveContentType(o.family)
vinfo, ok := r.ScannedCves[definition.Debian.CveID]
if !ok {
util.Log.Debugf("%s is newly detected by OVAL", definition.Debian.CveID)
@@ -86,7 +54,7 @@ func (o DebianBase) update(r *models.ScanResult, definition *ovalmodels.Definiti
}
} else {
cveContents := vinfo.CveContents
ctype := models.NewCveContentType(r.Family)
ctype := models.NewCveContentType(o.family)
if _, ok := vinfo.CveContents[ctype]; ok {
util.Log.Debugf("%s will be updated by OVAL", definition.Debian.CveID)
} else {
@@ -130,7 +98,9 @@ type Debian struct {
func NewDebian() Debian {
return Debian{
DebianBase{
family: config.Debian,
Base{
family: config.Debian,
},
},
}
}
@@ -170,7 +140,9 @@ type Ubuntu struct {
func NewUbuntu() Ubuntu {
return Ubuntu{
DebianBase{
family: config.Ubuntu,
Base{
family: config.Ubuntu,
},
},
}
}

153
oval/oval.go
View File

@@ -1,3 +1,20 @@
/* Vuls - Vulnerability Scanner
Copyright (C) 2016 Future Architect, Inc. Japan.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package oval
import (
@@ -7,14 +24,11 @@ import (
"strings"
"time"
"github.com/cenkalti/backoff"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
ver "github.com/knqyf263/go-deb-version"
"github.com/kotakanbe/goval-dictionary/db"
ovallog "github.com/kotakanbe/goval-dictionary/log"
ovalmodels "github.com/kotakanbe/goval-dictionary/models"
"github.com/parnurzeal/gorequest"
)
@@ -29,7 +43,9 @@ type Client interface {
}
// Base is a base struct
type Base struct{}
type Base struct {
family string
}
// CheckHTTPHealth do health check
func (b Base) CheckHTTPHealth() error {
@@ -135,132 +151,3 @@ func (b Base) isFetchViaHTTP() bool {
}
return false
}
type request struct {
pack models.Package
}
type response struct {
pack *models.Package
defs []ovalmodels.Definition
}
// getDefsByPackNameViaHTTP fetches OVAL information via HTTP
func getDefsByPackNameViaHTTP(r *models.ScanResult) (
relatedDefs []ovalmodels.Definition, err error) {
reqChan := make(chan request, len(r.Packages))
resChan := make(chan response, len(r.Packages))
errChan := make(chan error, len(r.Packages))
defer close(reqChan)
defer close(resChan)
defer close(errChan)
go func() {
for _, pack := range r.Packages {
reqChan <- request{
pack: pack,
}
}
}()
concurrency := 10
tasks := util.GenWorkers(concurrency)
for range r.Packages {
tasks <- func() {
select {
case req := <-reqChan:
url, err := util.URLPathJoin(
config.Conf.OvalDBURL,
"packs",
r.Family,
r.Release,
req.pack.Name,
)
if err != nil {
errChan <- err
} else {
util.Log.Debugf("HTTP Request to %s", url)
httpGet(url, &req.pack, resChan, errChan)
}
}
}
}
timeout := time.After(2 * 60 * time.Second)
var errs []error
for range r.Packages {
select {
case res := <-resChan:
current, _ := ver.NewVersion(fmt.Sprintf("%s-%s",
res.pack.Version, res.pack.Release))
for _, def := range res.defs {
for _, p := range def.AffectedPacks {
affected, _ := ver.NewVersion(p.Version)
if res.pack.Name != p.Name || !current.LessThan(affected) {
continue
}
relatedDefs = append(relatedDefs, def)
}
}
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
return nil, fmt.Errorf("Timeout Fetching OVAL")
}
}
if len(errs) != 0 {
return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs)
}
return
}
func httpGet(url string, pack *models.Package, resChan chan<- response, errChan chan<- error) {
var body string
var errs []error
var resp *http.Response
count, retryMax := 0, 3
f := func() (err error) {
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
resp, body, errs = gorequest.New().Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
count++
if count == retryMax {
return nil
}
return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v",
errs, url, resp)
}
return nil
}
notify := func(err error, t time.Duration) {
util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
}
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
if err != nil {
errChan <- fmt.Errorf("HTTP Error %s", err)
return
}
if count == retryMax {
errChan <- fmt.Errorf("HRetry count exceeded")
return
}
defs := []ovalmodels.Definition{}
if err := json.Unmarshal([]byte(body), &defs); err != nil {
errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s",
body, err)
return
}
resChan <- response{
pack: pack,
defs: defs,
}
}
func getPackages(r *models.ScanResult, d *ovalmodels.Definition) (names []string) {
for _, affectedPack := range d.AffectedPacks {
names = append(names, affectedPack.Name)
}
return
}

76
oval/redhat.go
View File

@@ -1,3 +1,20 @@
/* Vuls - Vulnerability Scanner
Copyright (C) 2016 Future Architect, Inc. Japan.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package oval
import (
@@ -8,16 +25,12 @@ import (
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
ver "github.com/knqyf263/go-rpm-version"
db "github.com/kotakanbe/goval-dictionary/db"
ovallog "github.com/kotakanbe/goval-dictionary/log"
ovalmodels "github.com/kotakanbe/goval-dictionary/models"
)
// RedHatBase is the base struct for RedHat and CentOS
type RedHatBase struct {
Base
family string
}
// FillWithOval returns scan result after updating CVE info by OVAL
@@ -54,7 +67,7 @@ func (o RedHatBase) FillWithOval(r *models.ScanResult) error {
// fillFromOvalDB returns scan result after updating CVE info by OVAL
func (o RedHatBase) fillFromOvalDB(r *models.ScanResult) error {
defs, err := o.getDefsByPackNameFromOvalDB(r.Release, r.Packages)
defs, err := getDefsByPackNameFromOvalDB(o.family, r.Release, r.Packages)
if err != nil {
return err
}
@@ -64,47 +77,6 @@ func (o RedHatBase) fillFromOvalDB(r *models.ScanResult) error {
return nil
}
func (o RedHatBase) getDefsByPackNameFromOvalDB(osRelease string,
packs models.Packages) (relatedDefs []ovalmodels.Definition, err error) {
ovallog.Initialize(config.Conf.LogDir)
path := config.Conf.OvalDBURL
if config.Conf.OvalDBType == "sqlite3" {
path = config.Conf.OvalDBPath
}
util.Log.Debugf("Open oval-dictionary db (%s): %s", config.Conf.OvalDBType, path)
var ovaldb db.DB
if ovaldb, err = db.NewDB(
o.family,
config.Conf.OvalDBType,
path,
config.Conf.DebugSQL,
); err != nil {
return
}
defer ovaldb.CloseDB()
for _, pack := range packs {
definitions, err := ovaldb.GetByPackName(osRelease, pack.Name)
if err != nil {
return nil, fmt.Errorf("Failed to get %s OVAL info by package name: %v", o.family, err)
}
for _, def := range definitions {
current := ver.NewVersion(fmt.Sprintf("%s-%s", pack.Version, pack.Release))
for _, p := range def.AffectedPacks {
if pack.Name != p.Name {
continue
}
affected := ver.NewVersion(p.Version)
if current.LessThan(affected) {
relatedDefs = append(relatedDefs, def)
}
}
}
}
return
}
func (o RedHatBase) update(r *models.ScanResult, definition *ovalmodels.Definition) {
ctype := models.NewCveContentType(o.family)
for _, cve := range definition.Advisory.Cves {
@@ -210,7 +182,9 @@ type RedHat struct {
func NewRedhat() RedHat {
return RedHat{
RedHatBase{
family: config.RedHat,
Base{
family: config.RedHat,
},
},
}
}
@@ -224,7 +198,9 @@ type CentOS struct {
func NewCentOS() CentOS {
return CentOS{
RedHatBase{
family: config.CentOS,
Base{
family: config.CentOS,
},
},
}
}
@@ -238,7 +214,9 @@ type Oracle struct {
func NewOracle() Oracle {
return Oracle{
RedHatBase{
family: config.Oracle,
Base{
family: config.Oracle,
},
},
}
}

233
oval/util.go Normal file
View File

@@ -0,0 +1,233 @@
/* Vuls - Vulnerability Scanner
Copyright (C) 2016 Future Architect, Inc. Japan.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package oval
import (
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/cenkalti/backoff"
"github.com/future-architect/vuls/config"
"github.com/future-architect/vuls/models"
"github.com/future-architect/vuls/util"
debver "github.com/knqyf263/go-deb-version"
rpmver "github.com/knqyf263/go-rpm-version"
"github.com/kotakanbe/goval-dictionary/db"
ovallog "github.com/kotakanbe/goval-dictionary/log"
ovalmodels "github.com/kotakanbe/goval-dictionary/models"
"github.com/parnurzeal/gorequest"
)
type request struct {
pack models.Package
}
type response struct {
pack *models.Package
defs []ovalmodels.Definition
}
// getDefsByPackNameViaHTTP fetches OVAL information via HTTP
func getDefsByPackNameViaHTTP(r *models.ScanResult) (
relatedDefs []ovalmodels.Definition, err error) {
reqChan := make(chan request, len(r.Packages))
resChan := make(chan response, len(r.Packages))
errChan := make(chan error, len(r.Packages))
defer close(reqChan)
defer close(resChan)
defer close(errChan)
go func() {
for _, pack := range r.Packages {
reqChan <- request{
pack: pack,
}
}
}()
concurrency := 10
tasks := util.GenWorkers(concurrency)
for range r.Packages {
tasks <- func() {
select {
case req := <-reqChan:
url, err := util.URLPathJoin(
config.Conf.OvalDBURL,
"packs",
r.Family,
r.Release,
req.pack.Name,
)
if err != nil {
errChan <- err
} else {
util.Log.Debugf("HTTP Request to %s", url)
httpGet(url, &req.pack, resChan, errChan)
}
}
}
}
timeout := time.After(2 * 60 * time.Second)
var errs []error
for range r.Packages {
select {
case res := <-resChan:
for _, def := range res.defs {
for _, p := range def.AffectedPacks {
if res.pack.Name != p.Name {
continue
}
if less, err := lessThan(r.Family, *res.pack, p); err != nil {
if !p.NotFixedYet {
util.Log.Debugf("Failed to parse versions: %s", err)
util.Log.Debugf("%#v\n%#v", *res.pack, p)
}
} else if less {
relatedDefs = append(relatedDefs, def)
}
}
}
case err := <-errChan:
errs = append(errs, err)
case <-timeout:
return nil, fmt.Errorf("Timeout Fetching OVAL")
}
}
if len(errs) != 0 {
return nil, fmt.Errorf("Failed to fetch OVAL. err: %v", errs)
}
return
}
func httpGet(url string, pack *models.Package, resChan chan<- response, errChan chan<- error) {
var body string
var errs []error
var resp *http.Response
count, retryMax := 0, 3
f := func() (err error) {
// resp, body, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End()
resp, body, errs = gorequest.New().Get(url).End()
if 0 < len(errs) || resp == nil || resp.StatusCode != 200 {
count++
if count == retryMax {
return nil
}
return fmt.Errorf("HTTP GET error: %v, url: %s, resp: %v",
errs, url, resp)
}
return nil
}
notify := func(err error, t time.Duration) {
util.Log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %s", t, err)
}
err := backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
if err != nil {
errChan <- fmt.Errorf("HTTP Error %s", err)
return
}
if count == retryMax {
errChan <- fmt.Errorf("HRetry count exceeded")
return
}
defs := []ovalmodels.Definition{}
if err := json.Unmarshal([]byte(body), &defs); err != nil {
errChan <- fmt.Errorf("Failed to Unmarshall. body: %s, err: %s",
body, err)
return
}
resChan <- response{
pack: pack,
defs: defs,
}
}
func getPackages(r *models.ScanResult, d *ovalmodels.Definition) (names []string) {
for _, affectedPack := range d.AffectedPacks {
names = append(names, affectedPack.Name)
}
return
}
func getDefsByPackNameFromOvalDB(family, osRelease string,
packs models.Packages) (relatedDefs []ovalmodels.Definition, err error) {
ovallog.Initialize(config.Conf.LogDir)
path := config.Conf.OvalDBURL
if config.Conf.OvalDBType == "sqlite3" {
path = config.Conf.OvalDBPath
}
util.Log.Debugf("Open oval-dictionary db (%s): %s", config.Conf.OvalDBType, path)
var ovaldb db.DB
if ovaldb, err = db.NewDB(
family,
config.Conf.OvalDBType,
path,
config.Conf.DebugSQL,
); err != nil {
return
}
defer ovaldb.CloseDB()
for _, pack := range packs {
definitions, err := ovaldb.GetByPackName(osRelease, pack.Name)
if err != nil {
return nil, fmt.Errorf("Failed to get %s OVAL info by package name: %v", family, err)
}
for _, def := range definitions {
for _, p := range def.AffectedPacks {
if pack.Name != p.Name {
continue
}
if less, err := lessThan(family, pack, p); err != nil {
if !p.NotFixedYet {
util.Log.Debugf("Failed to parse versions: %s", err)
util.Log.Debugf("%#v\n%#v", pack, p)
}
} else if less {
relatedDefs = append(relatedDefs, def)
}
}
}
}
return
}
func lessThan(family string, packA models.Package, packB ovalmodels.Package) (bool, error) {
switch family {
case config.Debian, config.Ubuntu:
vera, err := debver.NewVersion(packA.Version)
if err != nil {
return false, err
}
verb, err := debver.NewVersion(packB.Version)
if err != nil {
return false, err
}
return vera.LessThan(verb), nil
case config.RedHat, config.CentOS, config.Oracle:
vera := rpmver.NewVersion(fmt.Sprintf("%s-%s", packA.Version, packA.Release))
verb := rpmver.NewVersion(packB.Version)
return vera.LessThan(verb), nil
}
return false, fmt.Errorf("Package version comparison not supported: %s", family)
}