diff --git a/GNUmakefile b/GNUmakefile index 3cc9b7ae..d44cb749 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -15,7 +15,7 @@ clean SRCS = $(shell git ls-files '*.go') -PKGS = ./. ./config ./models ./report ./cveapi ./scan ./util ./commands ./cache +PKGS = ./. ./cache ./commands ./config ./models ./oval ./report ./scan ./util VERSION := $(shell git describe --tags --abbrev=0) REVISION := $(shell git rev-parse --short HEAD) LDFLAGS := -X 'main.version=$(VERSION)' \ diff --git a/commands/history.go b/commands/history.go index 1c6acf4b..2a6e02fb 100644 --- a/commands/history.go +++ b/commands/history.go @@ -27,6 +27,7 @@ import ( "strings" c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/report" "github.com/google/subcommands" ) @@ -68,9 +69,8 @@ func (p *HistoryCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{ c.Conf.DebugSQL = p.debugSQL c.Conf.ResultsDir = p.resultsDir - var err error - var dirs jsonDirs - if dirs, err = lsValidJSONDirs(); err != nil { + dirs, err := report.ListValidJSONDirs() + if err != nil { return subcommands.ExitFailure } for _, d := range dirs { diff --git a/commands/report.go b/commands/report.go index 1fe6c93b..442b936c 100644 --- a/commands/report.go +++ b/commands/report.go @@ -25,9 +25,7 @@ import ( "path/filepath" c "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/cveapi" "github.com/future-architect/vuls/models" - "github.com/future-architect/vuls/oval" "github.com/future-architect/vuls/report" "github.com/future-architect/vuls/util" "github.com/google/subcommands" @@ -290,6 +288,8 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} c.Conf.Lang = p.lang c.Conf.ResultsDir = p.resultsDir + c.Conf.RefreshCve = p.refreshCve + c.Conf.Diff = p.diff c.Conf.CveDBType = p.cvedbtype c.Conf.CveDBPath = p.cvedbpath c.Conf.CveDBURL = p.cvedbURL @@ -314,9 +314,9 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} var dir string var err error if p.diff { - dir, err = jsonDir([]string{}) + dir, err = report.JSONDir([]string{}) } else { - dir, err = jsonDir(f.Args()) + dir, err = report.JSONDir(f.Args()) } if err != nil { util.Log.Errorf("Failed to read from JSON: %s", err) @@ -385,7 +385,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} if !c.Conf.ValidateOnReport() { return subcommands.ExitUsageError } - if ok, err := cveapi.CveClient.CheckHealth(); !ok { + if ok, err := report.CveClient.CheckHealth(); !ok { util.Log.Errorf("CVE HTTP server is not running. err: %s", err) util.Log.Errorf("Run go-cve-dictionary as server mode before reporting or run with --cvedb-path option") return subcommands.ExitFailure @@ -398,90 +398,36 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } } - rs, err := loadScanResults(dir) - if err != nil { + var res models.ScanResults + if res, err = report.LoadScanResults(dir); err != nil { util.Log.Error(err) return subcommands.ExitFailure } util.Log.Infof("Loaded: %s", dir) - var results []models.ScanResult - for _, r := range rs { - if p.refreshCve || needToRefreshCve(r) { - util.Log.Debugf("need to refresh") - if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDBURL == "" { - if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) { - util.Log.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s", - c.Conf.CveDBPath) - return subcommands.ExitFailure - } - } - - if err := fillCveInfoFromOvalDB(&r); err != nil { - util.Log.Errorf("Failed to fill OVAL information: %s", err) - return subcommands.ExitFailure - } - - if err := fillCveInfoFromCveDB(&r); err != nil { - util.Log.Errorf("Failed to fill CVE information: %s", err) - return subcommands.ExitFailure - } - - r.Lang = c.Conf.Lang - if err := overwriteJSONFile(dir, r); err != nil { - util.Log.Errorf("Failed to write JSON: %s", err) - return subcommands.ExitFailure - } - results = append(results, r) - } else { - util.Log.Debugf("no need to refresh") - results = append(results, r) - } + //TODO dir + if res, err = report.FillCveInfos(res, dir); err != nil { + util.Log.Error(err) + return subcommands.ExitFailure } - if p.diff { - previous, err := loadPrevious(results) - if err != nil { - util.Log.Error(err) - return subcommands.ExitFailure - } - - diff, err := diff(results, previous) - if err != nil { - util.Log.Error(err) - return subcommands.ExitFailure - } - results = []models.ScanResult{} - for _, r := range diff { - if err := fillCveDetail(&r); err != nil { - util.Log.Error(err) - return subcommands.ExitFailure - } - results = append(results, r) - } - } - - var res models.ScanResults - for _, r := range results { - res = append(res, r.FilterByCvssOver(c.Conf.CvssScoreOver)) - - // TODO Add sort function to ScanResults - - //remove - // for _, vuln := range r.ScannedCves { - // // if _, ok := vuln.CveContents.Get(models.NewCveContentType(r.Family)); !ok { - // // pp.Printf("not in oval: %s %f\n%v\n", - // // vuln.CveID, vuln.CveContents.CvssV2Score(), vuln.Packages) - // // } else { - // // fmt.Printf(" in oval: %s %f\n", - // // vuln.CveID, vuln.CveContents.CvssV2Score()) - // // } - // // if vuln.CveContents.CvssV2Score() < 0.1 && - // // vuln.CveContents.CvssV3Score() < 0.1 { - // // pp.Println(vuln) - // // } - // } - } + // TODO Filter, Sort + // TODO Add sort function to ScanResults + //remove + // for _, vuln := range r.ScannedCves { + // // if _, ok := vuln.CveContents.Get(models.NewCveContentType(r.Family)); !ok { + // // pp.Printf("not in oval: %s %f\n%v\n", + // // vuln.CveID, vuln.CveContents.CvssV2Score(), vuln.Packages) + // // } else { + // // fmt.Printf(" in oval: %s %f\n", + // // vuln.CveID, vuln.CveContents.CvssV2Score()) + // // } + // // if vuln.CveContents.CvssV2Score() < 0.1 && + // // vuln.CveContents.CvssV3Score() < 0.1 { + // // pp.Println(vuln) + // // } + // } + // } for _, w := range reports { if err := w.Write(res...); err != nil { @@ -491,76 +437,3 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } return subcommands.ExitSuccess } - -// fillCveDetail fetches NVD, JVN from CVE Database, and then set to fields. -func fillCveDetail(r *models.ScanResult) error { - var cveIDs []string - for _, v := range r.ScannedCves { - cveIDs = append(cveIDs, v.CveID) - } - - ds, err := cveapi.CveClient.FetchCveDetails(cveIDs) - if err != nil { - return err - } - for _, d := range ds { - nvd := r.ConvertNvdToModel(d.CveID, d.Nvd) - jvn := r.ConvertJvnToModel(d.CveID, d.Jvn) - for cveID, vinfo := range r.ScannedCves { - if vinfo.CveID == d.CveID { - if vinfo.CveContents == nil { - vinfo.CveContents = models.CveContents{} - } - for _, con := range []models.CveContent{*nvd, *jvn} { - if !con.Empty() { - vinfo.CveContents[con.Type] = con - } - } - r.ScannedCves[cveID] = vinfo - break - } - } - } - //TODO Remove - // sort.Slice(r.ScannedCves, func(i, j int) bool { - // if r.ScannedCves[j].CveContents.CvssV2Score() == r.ScannedCves[i].CveContents.CvssV2Score() { - // return r.ScannedCves[j].CveContents.CvssV2Score() < r.ScannedCves[i].CveContents.CvssV2Score() - // } - // return r.ScannedCves[j].CveContents.CvssV2Score() < r.ScannedCves[i].CveContents.CvssV2Score() - // }) - return nil -} - -func fillCveInfoFromCveDB(r *models.ScanResult) error { - sInfo := c.Conf.Servers[r.ServerName] - if err := fillVulnByCpeNames(sInfo.CpeNames, r.ScannedCves); err != nil { - return err - } - if err := fillCveDetail(r); err != nil { - return err - } - return nil -} - -func fillCveInfoFromOvalDB(r *models.ScanResult) error { - var ovalClient oval.Client - switch r.Family { - case "debian": - ovalClient = oval.NewDebian() - case "ubuntu": - ovalClient = oval.NewUbuntu() - case "rhel": - ovalClient = oval.NewRedhat() - case "centos": - ovalClient = oval.NewCentOS() - case "amazon", "oraclelinux", "Raspbian", "FreeBSD": - //TODO implement OracleLinux - return nil - default: - return fmt.Errorf("Oval %s is not implemented yet", r.Family) - } - if err := ovalClient.FillCveInfoFromOvalDB(r); err != nil { - return err - } - return nil -} diff --git a/commands/tui.go b/commands/tui.go index cbf39a02..cd661a50 100644 --- a/commands/tui.go +++ b/commands/tui.go @@ -24,8 +24,6 @@ import ( "path/filepath" c "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/models" - "github.com/future-architect/vuls/report" "github.com/future-architect/vuls/util" "github.com/google/subcommands" ) @@ -146,40 +144,41 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s } c.Conf.Pipe = p.pipe - jsonDir, err := jsonDir(f.Args()) - if err != nil { - log.Errorf("Failed to read json dir under results: %s", err) - return subcommands.ExitFailure - } + // jsonDir, err := report.JSONDir(f.Args()) + // if err != nil { + // log.Errorf("Failed to read json dir under results: %s", err) + // return subcommands.ExitFailure + // } - results, err := loadScanResults(jsonDir) - if err != nil { - log.Errorf("Failed to read from JSON: %s", err) - return subcommands.ExitFailure - } + // results, err := report.LoadScanResults(jsonDir) + // if err != nil { + // log.Errorf("Failed to read from JSON: %s", err) + // return subcommands.ExitFailure + // } - var filledResults []models.ScanResult - for _, r := range results { - if p.refreshCve || needToRefreshCve(r) { - if c.Conf.CveDBType == "sqlite3" { - if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) { - log.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s", - c.Conf.CveDBPath) - return subcommands.ExitFailure - } - } + // var filledResults []models.ScanResult + // for _, r := range results { + // if p.refreshCve || needToRefreshCve(r) { + // if c.Conf.CveDBType == "sqlite3" { + // if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) { + // log.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s", + // c.Conf.CveDBPath) + // return subcommands.ExitFailure + // } + // } - if err := fillCveInfoFromCveDB(&r); err != nil { - log.Errorf("Failed to fill CVE information: %s", err) - return subcommands.ExitFailure - } + // if err := fillCveInfoFromCveDB(&r); err != nil { + // log.Errorf("Failed to fill CVE information: %s", err) + // return subcommands.ExitFailure + // } - if err := overwriteJSONFile(jsonDir, r); err != nil { - log.Errorf("Failed to write JSON: %s", err) - return subcommands.ExitFailure - } - } - filledResults = append(filledResults, r) - } - return report.RunTui(filledResults) + // if err := overwriteJSONFile(jsonDir, r); err != nil { + // log.Errorf("Failed to write JSON: %s", err) + // return subcommands.ExitFailure + // } + // } + // filledResults = append(filledResults, r) + // } + // return report.RunTui(filledResults) + return subcommands.ExitFailure } diff --git a/commands/util.go b/commands/util.go index 73d73001..d0a08b5f 100644 --- a/commands/util.go +++ b/commands/util.go @@ -18,307 +18,11 @@ along with this program. If not, see . package commands import ( - "encoding/json" "fmt" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "sort" - "strings" - "time" - c "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/cveapi" - "github.com/future-architect/vuls/models" - "github.com/future-architect/vuls/report" - "github.com/future-architect/vuls/util" "github.com/howeyc/gopass" ) -// jsonDirPattern is file name pattern of JSON directory -// 2016-11-16T10:43:28+09:00 -// 2016-11-16T10:43:28Z -var jsonDirPattern = regexp.MustCompile( - `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})$`) - -// JSONDirs is array of json files path. -type jsonDirs []string - -// getValidJSONDirs return valid json directory as array -// Returned array is sorted so that recent directories are at the head -func lsValidJSONDirs() (dirs jsonDirs, err error) { - var dirInfo []os.FileInfo - if dirInfo, err = ioutil.ReadDir(c.Conf.ResultsDir); err != nil { - err = fmt.Errorf("Failed to read %s: %s", c.Conf.ResultsDir, err) - return - } - for _, d := range dirInfo { - if d.IsDir() && jsonDirPattern.MatchString(d.Name()) { - jsonDir := filepath.Join(c.Conf.ResultsDir, d.Name()) - dirs = append(dirs, jsonDir) - } - } - sort.Slice(dirs, func(i, j int) bool { - return dirs[j] < dirs[i] - }) - return -} - -// jsonDir returns -// If there is an arg, check if it is a valid format and return the corresponding path under results. -// If arg passed via PIPE (such as history subcommand), return that path. -// Otherwise, returns the path of the latest directory -func jsonDir(args []string) (string, error) { - var err error - var dirs jsonDirs - - if 0 < len(args) { - if dirs, err = lsValidJSONDirs(); err != nil { - return "", err - } - - path := filepath.Join(c.Conf.ResultsDir, args[0]) - for _, d := range dirs { - ss := strings.Split(d, string(os.PathSeparator)) - timedir := ss[len(ss)-1] - if timedir == args[0] { - return path, nil - } - } - - return "", fmt.Errorf("Invalid path: %s", path) - } - - // PIPE - if c.Conf.Pipe { - bytes, err := ioutil.ReadAll(os.Stdin) - if err != nil { - return "", fmt.Errorf("Failed to read stdin: %s", err) - } - fields := strings.Fields(string(bytes)) - if 0 < len(fields) { - return filepath.Join(c.Conf.ResultsDir, fields[0]), nil - } - return "", fmt.Errorf("Stdin is invalid: %s", string(bytes)) - } - - // returns latest dir when no args or no PIPE - if dirs, err = lsValidJSONDirs(); err != nil { - return "", err - } - if len(dirs) == 0 { - return "", fmt.Errorf("No results under %s", - c.Conf.ResultsDir) - } - return dirs[0], nil -} - -// loadOneServerScanResult read JSON data of one server -func loadOneServerScanResult(jsonFile string) (result models.ScanResult, err error) { - var data []byte - if data, err = ioutil.ReadFile(jsonFile); err != nil { - err = fmt.Errorf("Failed to read %s: %s", jsonFile, err) - return - } - if json.Unmarshal(data, &result) != nil { - err = fmt.Errorf("Failed to parse %s: %s", jsonFile, err) - } - return -} - -// loadScanResults read JSON data -func loadScanResults(jsonDir string) (results models.ScanResults, err error) { - var files []os.FileInfo - if files, err = ioutil.ReadDir(jsonDir); err != nil { - return nil, fmt.Errorf("Failed to read %s: %s", jsonDir, err) - } - for _, f := range files { - if filepath.Ext(f.Name()) != ".json" || strings.HasSuffix(f.Name(), "_diff.json") { - continue - } - - var r models.ScanResult - path := filepath.Join(jsonDir, f.Name()) - if r, err = loadOneServerScanResult(path); err != nil { - return nil, err - } - - results = append(results, r) - } - if len(results) == 0 { - return nil, fmt.Errorf("There is no json file under %s", jsonDir) - } - return -} - -func loadPrevious(current models.ScanResults) (previous models.ScanResults, err error) { - var dirs jsonDirs - if dirs, err = lsValidJSONDirs(); err != nil { - return - } - - for _, result := range current { - for _, dir := range dirs[1:] { - var r models.ScanResult - path := filepath.Join(dir, result.ServerName+".json") - if r, err = loadOneServerScanResult(path); err != nil { - continue - } - if r.Family == result.Family && r.Release == result.Release { - previous = append(previous, r) - util.Log.Infof("Privious json found: %s", path) - break - } - } - } - return previous, nil -} - -func diff(curResults, preResults models.ScanResults) (diffed models.ScanResults, err error) { - for _, current := range curResults { - found := false - var previous models.ScanResult - for _, r := range preResults { - if current.ServerName == r.ServerName { - found = true - previous = r - break - } - } - - if found { - current.ScannedCves = getDiffCves(previous, current) - packages := models.Packages{} - for _, s := range current.ScannedCves { - for _, name := range s.PackageNames { - p := current.Packages[name] - packages[name] = p - } - } - current.Packages = packages - } - - diffed = append(diffed, current) - } - return diffed, err -} - -func getDiffCves(previous, current models.ScanResult) models.VulnInfos { - previousCveIDsSet := map[string]bool{} - for _, previousVulnInfo := range previous.ScannedCves { - previousCveIDsSet[previousVulnInfo.CveID] = true - } - - new := models.VulnInfos{} - updated := models.VulnInfos{} - for _, v := range current.ScannedCves { - if previousCveIDsSet[v.CveID] { - if isCveInfoUpdated(v.CveID, previous, current) { - updated[v.CveID] = v - } - } else { - new[v.CveID] = v - } - } - - for cveID, vuln := range new { - updated[cveID] = vuln - } - return updated -} - -func isCveInfoUpdated(cveID string, previous, current models.ScanResult) bool { - cTypes := []models.CveContentType{ - models.NVD, - models.JVN, - models.NewCveContentType(current.Family), - } - - prevLastModified := map[models.CveContentType]time.Time{} - for _, c := range previous.ScannedCves { - if cveID == c.CveID { - for _, cType := range cTypes { - content, _ := c.CveContents[cType] - prevLastModified[cType] = content.LastModified - } - break - } - } - - curLastModified := map[models.CveContentType]time.Time{} - for _, c := range current.ScannedCves { - if cveID == c.CveID { - for _, cType := range cTypes { - content, _ := c.CveContents[cType] - curLastModified[cType] = content.LastModified - } - break - } - } - for _, cType := range cTypes { - if equal := prevLastModified[cType].Equal(curLastModified[cType]); !equal { - return true - } - } - return false -} - -func overwriteJSONFile(dir string, r models.ScanResult) error { - before := c.Conf.FormatJSON - beforeDiff := c.Conf.Diff - c.Conf.FormatJSON = true - c.Conf.Diff = false - w := report.LocalFileWriter{CurrentDir: dir} - if err := w.Write(r); err != nil { - return fmt.Errorf("Failed to write summary report: %s", err) - } - c.Conf.FormatJSON = before - c.Conf.Diff = beforeDiff - return nil -} - -func fillVulnByCpeNames(cpeNames []string, scannedVulns models.VulnInfos) error { - for _, name := range cpeNames { - details, err := cveapi.CveClient.FetchCveDetailsByCpeName(name) - if err != nil { - return err - } - for _, detail := range details { - if val, ok := scannedVulns[detail.CveID]; ok { - names := val.CpeNames - names = util.AppendIfMissing(names, name) - val.CpeNames = names - val.Confidence = models.CpeNameMatch - scannedVulns[detail.CveID] = val - } else { - v := models.VulnInfo{ - CveID: detail.CveID, - CpeNames: []string{name}, - Confidence: models.CpeNameMatch, - } - //TODO - // v.NilToEmpty() - scannedVulns[detail.CveID] = v - } - } - } - return nil -} - -func needToRefreshCve(r models.ScanResult) bool { - if r.Lang != c.Conf.Lang { - return true - } - - for _, cve := range r.ScannedCves { - if 0 < len(cve.CveContents) { - return false - } - } - return true -} - func getPasswd(prompt string) (string, error) { for { fmt.Print(prompt) diff --git a/commands/util_test.go b/commands/util_test.go index 25d872fc..e9ab19df 100644 --- a/commands/util_test.go +++ b/commands/util_test.go @@ -16,329 +16,3 @@ along with this program. If not, see . */ package commands - -import ( - "reflect" - "testing" - "time" - - "github.com/future-architect/vuls/models" - "github.com/k0kubun/pp" -) - -func TestIsCveInfoUpdated(t *testing.T) { - f := "2006-01-02" - old, _ := time.Parse(f, "2015-12-15") - new, _ := time.Parse(f, "2015-12-16") - - type In struct { - cveID string - cur models.ScanResult - prev models.ScanResult - } - var tests = []struct { - in In - expected bool - }{ - // NVD compare non-initialized times - { - in: In{ - cveID: "CVE-2017-0001", - cur: models.ScanResult{ - ScannedCves: models.VulnInfos{ - "CVE-2017-0001": { - CveID: "CVE-2017-0001", - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.NVD, - CveID: "CVE-2017-0001", - LastModified: time.Time{}, - }, - ), - }, - }, - }, - prev: models.ScanResult{ - ScannedCves: models.VulnInfos{ - "CVE-2017-0001": { - CveID: "CVE-2017-0001", - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.NVD, - CveID: "CVE-2017-0001", - LastModified: time.Time{}, - }, - ), - }, - }, - }, - }, - expected: false, - }, - // JVN not updated - { - in: In{ - cveID: "CVE-2017-0002", - cur: models.ScanResult{ - ScannedCves: models.VulnInfos{ - "CVE-2017-0002": { - CveID: "CVE-2017-0002", - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.NVD, - CveID: "CVE-2017-0002", - LastModified: old, - }, - ), - }, - }, - }, - prev: models.ScanResult{ - ScannedCves: models.VulnInfos{ - "CVE-2017-0002": { - CveID: "CVE-2017-0002", - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.NVD, - CveID: "CVE-2017-0002", - LastModified: old, - }, - ), - }, - }, - }, - }, - expected: false, - }, - // OVAL updated - { - in: In{ - cveID: "CVE-2017-0003", - cur: models.ScanResult{ - Family: "ubuntu", - ScannedCves: models.VulnInfos{ - "CVE-2017-0003": { - CveID: "CVE-2017-0003", - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.NVD, - CveID: "CVE-2017-0002", - LastModified: new, - }, - ), - }, - }, - }, - prev: models.ScanResult{ - Family: "ubuntu", - ScannedCves: models.VulnInfos{ - "CVE-2017-0003": { - CveID: "CVE-2017-0003", - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.NVD, - CveID: "CVE-2017-0002", - LastModified: old, - }, - ), - }, - }, - }, - }, - expected: true, - }, - // OVAL newly detected - { - in: In{ - cveID: "CVE-2017-0004", - cur: models.ScanResult{ - Family: "redhat", - ScannedCves: models.VulnInfos{ - "CVE-2017-0004": { - CveID: "CVE-2017-0004", - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.NVD, - CveID: "CVE-2017-0002", - LastModified: old, - }, - ), - }, - }, - }, - prev: models.ScanResult{ - Family: "redhat", - ScannedCves: models.VulnInfos{}, - }, - }, - expected: true, - }, - } - for i, tt := range tests { - actual := isCveInfoUpdated(tt.in.cveID, tt.in.prev, tt.in.cur) - if actual != tt.expected { - t.Errorf("[%d] actual: %t, expected: %t", i, actual, tt.expected) - } - } -} - -func TestDiff(t *testing.T) { - atCurrent, _ := time.Parse("2006-01-02", "2014-12-31") - atPrevious, _ := time.Parse("2006-01-02", "2014-11-31") - var tests = []struct { - inCurrent models.ScanResults - inPrevious models.ScanResults - out models.ScanResult - }{ - { - inCurrent: models.ScanResults{ - { - ScannedAt: atCurrent, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - ScannedCves: models.VulnInfos{ - "CVE-2012-6702": { - CveID: "CVE-2012-6702", - PackageNames: []string{"libexpat1"}, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - "CVE-2014-9761": { - CveID: "CVE-2014-9761", - PackageNames: []string{"libc-bin"}, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - }, - Packages: models.Packages{}, - Errors: []string{}, - Optional: [][]interface{}{}, - }, - }, - inPrevious: models.ScanResults{ - { - ScannedAt: atPrevious, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - ScannedCves: models.VulnInfos{ - "CVE-2012-6702": { - CveID: "CVE-2012-6702", - PackageNames: []string{"libexpat1"}, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - "CVE-2014-9761": { - CveID: "CVE-2014-9761", - PackageNames: []string{"libc-bin"}, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - }, - Packages: models.Packages{}, - Errors: []string{}, - Optional: [][]interface{}{}, - }, - }, - out: models.ScanResult{ - ScannedAt: atCurrent, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - Packages: models.Packages{}, - ScannedCves: models.VulnInfos{}, - Errors: []string{}, - Optional: [][]interface{}{}, - }, - }, - { - inCurrent: models.ScanResults{ - { - ScannedAt: atCurrent, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - ScannedCves: models.VulnInfos{ - "CVE-2016-6662": { - CveID: "CVE-2016-6662", - PackageNames: []string{"mysql-libs"}, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - }, - Packages: models.Packages{ - "mysql-libs": { - Name: "mysql-libs", - Version: "5.1.73", - Release: "7.el6", - NewVersion: "5.1.73", - NewRelease: "8.el6_8", - Repository: "", - Changelog: models.Changelog{ - Contents: "", - Method: "", - }, - }, - }, - }, - }, - inPrevious: models.ScanResults{ - { - ScannedAt: atPrevious, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - ScannedCves: models.VulnInfos{}, - }, - }, - out: models.ScanResult{ - ScannedAt: atCurrent, - ServerName: "u16", - Family: "ubuntu", - Release: "16.04", - ScannedCves: models.VulnInfos{ - "CVE-2016-6662": { - CveID: "CVE-2016-6662", - PackageNames: []string{"mysql-libs"}, - DistroAdvisories: []models.DistroAdvisory{}, - CpeNames: []string{}, - }, - }, - Packages: models.Packages{ - "mysql-libs": { - Name: "mysql-libs", - Version: "5.1.73", - Release: "7.el6", - NewVersion: "5.1.73", - NewRelease: "8.el6_8", - Repository: "", - Changelog: models.Changelog{ - Contents: "", - Method: "", - }, - }, - }, - }, - }, - } - - for i, tt := range tests { - diff, _ := diff(tt.inCurrent, tt.inPrevious) - for _, actual := range diff { - if !reflect.DeepEqual(actual.ScannedCves, tt.out.ScannedCves) { - h := pp.Sprint(actual.ScannedCves) - x := pp.Sprint(tt.out.ScannedCves) - t.Errorf("[%d] cves actual: \n %s \n expected: \n %s", i, h, x) - } - - for j := range tt.out.Packages { - if !reflect.DeepEqual(tt.out.Packages[j], actual.Packages[j]) { - h := pp.Sprint(tt.out.Packages[j]) - x := pp.Sprint(actual.Packages[j]) - t.Errorf("[%d] packages actual: \n %s \n expected: \n %s", i, x, h) - } - } - } - } -} diff --git a/config/config.go b/config/config.go index db813314..243efbe9 100644 --- a/config/config.go +++ b/config/config.go @@ -61,6 +61,8 @@ type Config struct { OvalDBType string OvalDBPath string + RefreshCve bool + FormatXML bool FormatJSON bool FormatOneEMail bool diff --git a/models/cvecontents.go b/models/cvecontents.go new file mode 100644 index 00000000..33ab332f --- /dev/null +++ b/models/cvecontents.go @@ -0,0 +1,527 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package models + +import ( + "fmt" + "strings" + "time" +) + +// CveContents has CveContent +type CveContents map[CveContentType]CveContent + +// NewCveContents create CveContents +func NewCveContents(conts ...CveContent) CveContents { + m := map[CveContentType]CveContent{} + for _, cont := range conts { + m[cont.Type] = cont + } + return m +} + +// CveContentStr has CveContentType and Value +type CveContentStr struct { + Type CveContentType + Value string +} + +// Except returns CveContents except given keys for enumeration +func (v CveContents) Except(exceptCtypes ...CveContentType) (values CveContents) { + for ctype, content := range v { + found := false + for _, exceptCtype := range exceptCtypes { + if ctype == exceptCtype { + found = true + break + } + } + if !found { + values[ctype] = content + } + } + return +} + +// CveContentCvss2 has CveContentType and Cvss2 +type CveContentCvss2 struct { + Type CveContentType + Value Cvss2 +} + +// Cvss2 has CVSS v2 +type Cvss2 struct { + Score float64 + Vector string + Severity string +} + +// Format CVSS Score and Vector +func (c Cvss2) Format() string { + return fmt.Sprintf("%3.1f/%s", c.Score, c.Vector) +} + +func cvss2ScoreToSeverity(score float64) string { + if 7.0 <= score { + return "HIGH" + } else if 4.0 <= score { + return "MEDIUM" + } + return "LOW" +} + +// Cvss2Scores returns CVSS V2 Scores +func (v CveContents) Cvss2Scores() (values []CveContentCvss2) { + order := []CveContentType{NVD, RedHat, JVN} + for _, ctype := range order { + if cont, found := v[ctype]; found && 0 < cont.Cvss2Score { + // https://nvd.nist.gov/vuln-metrics/cvss + sev := cont.Severity + if ctype == NVD { + sev = cvss2ScoreToSeverity(cont.Cvss2Score) + } + values = append(values, CveContentCvss2{ + Type: ctype, + Value: Cvss2{ + Score: cont.Cvss2Score, + Vector: cont.Cvss2Vector, + Severity: sev, + }, + }) + } + } + return +} + +// MaxCvss2Score returns Max CVSS V2 Score +func (v CveContents) MaxCvss2Score() CveContentCvss2 { + //TODO Severity Ubuntu, Debian... + order := []CveContentType{NVD, RedHat, JVN} + max := 0.0 + value := CveContentCvss2{ + Type: Unknown, + Value: Cvss2{}, + } + for _, ctype := range order { + if cont, found := v[ctype]; found && max < cont.Cvss2Score { + // https://nvd.nist.gov/vuln-metrics/cvss + sev := cont.Severity + if ctype == NVD { + sev = cvss2ScoreToSeverity(cont.Cvss2Score) + } + value = CveContentCvss2{ + Type: ctype, + Value: Cvss2{ + Score: cont.Cvss2Score, + Vector: cont.Cvss2Vector, + Severity: sev, + }, + } + max = cont.Cvss2Score + } + } + return value +} + +// CveContentCvss3 has CveContentType and Cvss3 +type CveContentCvss3 struct { + Type CveContentType + Value Cvss3 +} + +// Cvss3 has CVSS v3 Score, Vector and Severity +type Cvss3 struct { + Score float64 + Vector string + Severity string +} + +// Format CVSS Score and Vector +func (c Cvss3) Format() string { + return fmt.Sprintf("%3.1f/CVSS:3.0/%s", c.Score, c.Vector) +} + +func cvss3ScoreToSeverity(score float64) string { + if 9.0 <= score { + return "CRITICAL" + } else if 7.0 <= score { + return "HIGH" + } else if 4.0 <= score { + return "MEDIUM" + } + return "LOW" +} + +// Cvss3Scores returns CVSS V3 Score +func (v CveContents) Cvss3Scores() (values []CveContentCvss3) { + //TODO Severity Ubuntu, Debian... + order := []CveContentType{RedHat} + for _, ctype := range order { + if cont, found := v[ctype]; found && 0 < cont.Cvss3Score { + // https://nvd.nist.gov/vuln-metrics/cvss + sev := cont.Severity + if ctype == NVD { + sev = cvss3ScoreToSeverity(cont.Cvss2Score) + } + values = append(values, CveContentCvss3{ + Type: ctype, + Value: Cvss3{ + Score: cont.Cvss3Score, + Vector: cont.Cvss3Vector, + Severity: sev, + }, + }) + } + } + return +} + +// MaxCvss3Score returns Max CVSS V3 Score +func (v CveContents) MaxCvss3Score() CveContentCvss3 { + //TODO Severity Ubuntu, Debian... + order := []CveContentType{RedHat} + max := 0.0 + value := CveContentCvss3{ + Type: Unknown, + Value: Cvss3{}, + } + for _, ctype := range order { + if cont, found := v[ctype]; found && max < cont.Cvss3Score { + // https://nvd.nist.gov/vuln-metrics/cvss + sev := cont.Severity + if ctype == NVD { + sev = cvss3ScoreToSeverity(cont.Cvss2Score) + } + value = CveContentCvss3{ + Type: ctype, + Value: Cvss3{ + Score: cont.Cvss3Score, + Vector: cont.Cvss3Vector, + Severity: sev, + }, + } + max = cont.Cvss3Score + } + } + return value +} + +// FormatMaxCvssScore returns Max CVSS Score +func (v CveContents) FormatMaxCvssScore() string { + v2Max := v.MaxCvss2Score() + v3Max := v.MaxCvss3Score() + if v2Max.Value.Score <= v3Max.Value.Score { + return fmt.Sprintf("%3.1f %s (%s)", + v3Max.Value.Score, + strings.ToUpper(v3Max.Value.Severity), + v3Max.Type) + } + return fmt.Sprintf("%3.1f %s (%s)", + v2Max.Value.Score, + strings.ToUpper(v2Max.Value.Severity), + v2Max.Type) +} + +// Titles returns tilte (TUI) +func (v CveContents) Titles(lang, myFamily string) (values []CveContentStr) { + if lang == "ja" { + if cont, found := v[JVN]; found && 0 < len(cont.Title) { + values = append(values, CveContentStr{JVN, cont.Title}) + } + } + + order := CveContentTypes{NVD, NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order, JVN)...)...) + for _, ctype := range order { + // Only JVN has meaningful title. so return first 100 char of summary + if cont, found := v[ctype]; found && 0 < len(cont.Summary) { + summary := strings.Replace(cont.Summary, "\n", " ", -1) + index := 75 + if len(summary) < index { + index = len(summary) + } + values = append(values, CveContentStr{ + Type: ctype, + Value: summary[0:index] + "...", + }) + } + } + + if len(values) == 0 { + values = []CveContentStr{{ + Type: Unknown, + Value: "-", + }} + } + return +} + +// Summaries returns summaries +func (v CveContents) Summaries(lang, myFamily string) (values []CveContentStr) { + if lang == "ja" { + if cont, found := v[JVN]; found && 0 < len(cont.Summary) { + summary := cont.Title + summary += "\n" + strings.Replace( + strings.Replace(cont.Summary, "\n", " ", -1), "\r", " ", -1) + values = append(values, CveContentStr{JVN, summary}) + } + } + + order := CveContentTypes{NVD, NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order, JVN)...)...) + for _, ctype := range order { + if cont, found := v[ctype]; found && 0 < len(cont.Summary) { + summary := strings.Replace(cont.Summary, "\n", " ", -1) + values = append(values, CveContentStr{ + Type: ctype, + Value: summary, + }) + } + } + + if len(values) == 0 { + values = []CveContentStr{{ + Type: Unknown, + Value: "-", + }} + } + return +} + +// SourceLinks returns link of source +func (v CveContents) SourceLinks(lang, myFamily, cveID string) (values []CveContentStr) { + if lang == "ja" { + if cont, found := v[JVN]; found && !cont.Empty() { + values = append(values, CveContentStr{JVN, cont.SourceLink}) + } + } + + order := CveContentTypes{NVD, NewCveContentType(myFamily)} + for _, ctype := range order { + if cont, found := v[ctype]; found { + values = append(values, CveContentStr{ctype, cont.SourceLink}) + } + } + + if len(values) == 0 { + return []CveContentStr{{ + Type: NVD, + Value: "https://nvd.nist.gov/vuln/detail/" + cveID, + }} + } + return values +} + +// VendorLink returns link of source +func (v CveContents) VendorLink(myFamily string) CveContentStr { + ctype := NewCveContentType(myFamily) + if cont, ok := v[ctype]; ok { + return CveContentStr{ctype, cont.SourceLink} + } + return CveContentStr{ctype, ""} +} + +// Severities returns Severities +// func (v CveContents) Severities(myFamily string) (values []CveContentValue) { +// order := CveContentTypes{NVD, NewCveContentType(myFamily)} +// order = append(order, AllCveContetTypes.Except(append(order)...)...) + +// for _, ctype := range order { +// if cont, found := v[ctype]; found && 0 < len(cont.Severity) { +// values = append(values, CveContentValue{ +// Type: ctype, +// Value: cont.Severity, +// }) +// } +// } +// return +// } + +// CveContentCpes has CveContentType and Value +type CveContentCpes struct { + Type CveContentType + Value []Cpe +} + +// Cpes returns affected CPEs of this Vulnerability +func (v CveContents) Cpes(myFamily string) (values []CveContentCpes) { + order := CveContentTypes{NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order)...)...) + + for _, ctype := range order { + if cont, found := v[ctype]; found && 0 < len(cont.Cpes) { + values = append(values, CveContentCpes{ + Type: ctype, + Value: cont.Cpes, + }) + } + } + return +} + +// CveContentRefs has CveContentType and Cpes +type CveContentRefs struct { + Type CveContentType + Value []Reference +} + +// References returns References +func (v CveContents) References(myFamily string) (values []CveContentRefs) { + order := CveContentTypes{NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order)...)...) + + for _, ctype := range order { + if cont, found := v[ctype]; found && 0 < len(cont.References) { + values = append(values, CveContentRefs{ + Type: ctype, + Value: cont.References, + }) + } + } + return +} + +// CweIDs returns related CweIDs of the vulnerability +func (v CveContents) CweIDs(myFamily string) (values []CveContentStr) { + order := CveContentTypes{NewCveContentType(myFamily)} + order = append(order, AllCveContetTypes.Except(append(order)...)...) + + for _, ctype := range order { + if cont, found := v[ctype]; found && 0 < len(cont.CweID) { + // RedHat's OVAL sometimes contains multiple CWE-IDs separated by spaces + for _, cweID := range strings.Fields(cont.CweID) { + values = append(values, CveContentStr{ + Type: ctype, + Value: cweID, + }) + } + } + } + return +} + +// CveContent has abstraction of various vulnerability information +type CveContent struct { + Type CveContentType + CveID string + Title string + Summary string + Severity string + Cvss2Score float64 + Cvss2Vector string + Cvss3Score float64 + Cvss3Vector string + SourceLink string + Cpes []Cpe + References References + CweID string + Published time.Time + LastModified time.Time +} + +// Empty checks the content is empty +func (c CveContent) Empty() bool { + return c.Summary == "" +} + +// CveContentType is a source of CVE information +type CveContentType string + +// NewCveContentType create CveContentType +func NewCveContentType(name string) CveContentType { + switch name { + case "nvd": + return NVD + case "jvn": + return JVN + case "redhat", "centos": + return RedHat + case "ubuntu": + return Ubuntu + case "debian": + return Debian + default: + return Unknown + } +} + +const ( + // NVD is NVD + NVD CveContentType = "nvd" + + // JVN is JVN + JVN CveContentType = "jvn" + + // RedHat is RedHat + RedHat CveContentType = "redhat" + + // Debian is Debian + Debian CveContentType = "debian" + + // Ubuntu is Ubuntu + Ubuntu CveContentType = "ubuntu" + + // Unknown is Unknown + Unknown CveContentType = "unknown" +) + +// CveContentTypes has slide of CveContentType +type CveContentTypes []CveContentType + +// AllCveContetTypes has all of CveContentTypes +var AllCveContetTypes = CveContentTypes{NVD, JVN, RedHat, Debian, Ubuntu} + +// Except returns CveContentTypes except for given args +func (c CveContentTypes) Except(excepts ...CveContentType) (excepted CveContentTypes) { + for _, ctype := range c { + found := false + for _, except := range excepts { + if ctype == except { + found = true + break + } + } + if !found { + excepted = append(excepted, ctype) + } + } + return +} + +// Cpe is Common Platform Enumeration +type Cpe struct { + CpeName string +} + +// References is a slice of Reference +type References []Reference + +// Find elements that matches the function passed in argument +func (r References) Find(f func(r Reference) bool) (refs []Reference) { + for _, rr := range r { + refs = append(refs, rr) + } + return +} + +// Reference has a related link of the CVE +type Reference struct { + Source string + Link string + RefID string +} diff --git a/models/cvecontents_test.go b/models/cvecontents_test.go new file mode 100644 index 00000000..fced7012 --- /dev/null +++ b/models/cvecontents_test.go @@ -0,0 +1,17 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package models diff --git a/models/models.go b/models/models.go index c92f8164..bd7ab57f 100644 --- a/models/models.go +++ b/models/models.go @@ -17,1017 +17,5 @@ along with this program. If not, see . package models -import ( - "bytes" - "fmt" - "strings" - "time" - - "github.com/future-architect/vuls/config" - cvedict "github.com/kotakanbe/go-cve-dictionary/models" -) - // JSONVersion is JSON Version const JSONVersion = "0.3.0" - -// ScanResults is slice of ScanResult. -type ScanResults []ScanResult - -//TODO -// // Len implement Sort Interface -// func (s ScanResults) Len() int { -// return len(s) -// } - -// // Swap implement Sort Interface -// func (s ScanResults) Swap(i, j int) { -// s[i], s[j] = s[j], s[i] -// } - -// // Less implement Sort Interface -// func (s ScanResults) Less(i, j int) bool { -// if s[i].ServerName == s[j].ServerName { -// return s[i].Container.ContainerID < s[i].Container.ContainerID -// } -// return s[i].ServerName < s[j].ServerName -// } - -// ScanResult has the result of scanned CVE information. -type ScanResult struct { - ScannedAt time.Time - JSONVersion string - Lang string - ServerName string // TOML Section key - Family string - Release string - Container Container - Platform Platform - - // Scanned Vulns by SSH scan + CPE + OVAL - ScannedCves VulnInfos - - Packages Packages - Errors []string - Optional [][]interface{} -} - -// ConvertNvdToModel convert NVD to CveContent -func (r ScanResult) ConvertNvdToModel(cveID string, nvd cvedict.Nvd) *CveContent { - var cpes []Cpe - for _, c := range nvd.Cpes { - cpes = append(cpes, Cpe{CpeName: c.CpeName}) - } - - var refs []Reference - for _, r := range nvd.References { - refs = append(refs, Reference{ - Link: r.Link, - Source: r.Source, - }) - } - - validVec := true - for _, v := range []string{ - nvd.AccessVector, - nvd.AccessComplexity, - nvd.Authentication, - nvd.ConfidentialityImpact, - nvd.IntegrityImpact, - nvd.AvailabilityImpact, - } { - if len(v) == 0 { - validVec = false - } - } - - vector := "" - if validVec { - vector = fmt.Sprintf("AV:%s/AC:%s/Au:%s/C:%s/I:%s/A:%s", - string(nvd.AccessVector[0]), - string(nvd.AccessComplexity[0]), - string(nvd.Authentication[0]), - string(nvd.ConfidentialityImpact[0]), - string(nvd.IntegrityImpact[0]), - string(nvd.AvailabilityImpact[0])) - } - - //TODO CVSSv3 - return &CveContent{ - Type: NVD, - CveID: cveID, - Summary: nvd.Summary, - Cvss2Score: nvd.Score, - Cvss2Vector: vector, - Severity: "", // severity is not contained in NVD - SourceLink: "https://nvd.nist.gov/vuln/detail/" + cveID, - Cpes: cpes, - CweID: nvd.CweID, - References: refs, - Published: nvd.PublishedDate, - LastModified: nvd.LastModifiedDate, - } -} - -// ConvertJvnToModel convert JVN to CveContent -func (r ScanResult) ConvertJvnToModel(cveID string, jvn cvedict.Jvn) *CveContent { - var cpes []Cpe - for _, c := range jvn.Cpes { - cpes = append(cpes, Cpe{CpeName: c.CpeName}) - } - - refs := []Reference{} - for _, r := range jvn.References { - refs = append(refs, Reference{ - Link: r.Link, - Source: r.Source, - }) - } - - vector := strings.TrimSuffix(strings.TrimPrefix(jvn.Vector, "("), ")") - return &CveContent{ - Type: JVN, - CveID: cveID, - Title: jvn.Title, - Summary: jvn.Summary, - Severity: jvn.Severity, - Cvss2Score: jvn.Score, - Cvss2Vector: vector, - SourceLink: jvn.JvnLink, - Cpes: cpes, - References: refs, - Published: jvn.PublishedDate, - LastModified: jvn.LastModifiedDate, - } -} - -// FilterByCvssOver is filter function. -func (r ScanResult) FilterByCvssOver(over float64) ScanResult { - // TODO: Set correct default value - if over == 0 { - over = -1.1 - } - - // TODO: Filter by ignore cves??? - filtered := r.ScannedCves.Find(func(v VulnInfo) bool { - values := v.CveContents.Cvss2Scores() - for _, v := range values { - score := v.Value.Score - if over <= score { - return true - } - } - return false - }) - - copiedScanResult := r - copiedScanResult.ScannedCves = filtered - return copiedScanResult -} - -// ReportFileName returns the filename on localhost without extention -func (r ScanResult) ReportFileName() (name string) { - if len(r.Container.ContainerID) == 0 { - return fmt.Sprintf("%s", r.ServerName) - } - return fmt.Sprintf("%s@%s", r.Container.Name, r.ServerName) -} - -// ReportKeyName returns the name of key on S3, Azure-Blob without extention -func (r ScanResult) ReportKeyName() (name string) { - timestr := r.ScannedAt.Format(time.RFC3339) - if len(r.Container.ContainerID) == 0 { - return fmt.Sprintf("%s/%s", timestr, r.ServerName) - } - return fmt.Sprintf("%s/%s@%s", timestr, r.Container.Name, r.ServerName) -} - -// ServerInfo returns server name one line -func (r ScanResult) ServerInfo() string { - if len(r.Container.ContainerID) == 0 { - return fmt.Sprintf("%s (%s%s)", - r.ServerName, r.Family, r.Release) - } - return fmt.Sprintf( - "%s / %s (%s%s) on %s", - r.Container.Name, - r.Container.ContainerID, - r.Family, - r.Release, - r.ServerName, - ) -} - -// ServerInfoTui returns server infromation for TUI sidebar -func (r ScanResult) ServerInfoTui() string { - if len(r.Container.ContainerID) == 0 { - return fmt.Sprintf("%s (%s%s)", - r.ServerName, r.Family, r.Release) - } - return fmt.Sprintf( - "|-- %s (%s%s)", - r.Container.Name, - r.Family, - r.Release, - // r.Container.ContainerID, - ) -} - -// FormatServerName returns server and container name -func (r ScanResult) FormatServerName() string { - if len(r.Container.ContainerID) == 0 { - return r.ServerName - } - return fmt.Sprintf("%s@%s", - r.Container.Name, r.ServerName) -} - -// CveSummary summarize the number of CVEs group by CVSSv2 Severity -func (r ScanResult) CveSummary(ignoreUnscoreCves bool) string { - var high, medium, low, unknown int - for _, vInfo := range r.ScannedCves { - score := vInfo.CveContents.MaxCvss2Score().Value.Score - if score < 0.1 { - score = vInfo.CveContents.MaxCvss3Score().Value.Score - } - switch { - case 7.0 <= score: - high++ - case 4.0 <= score: - medium++ - case 0 < score: - low++ - default: - unknown++ - } - } - - if ignoreUnscoreCves { - return fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d)", - high+medium+low, high, medium, low) - } - return fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d ?:%d)", - high+medium+low+unknown, high, medium, low, unknown) -} - -// FormatTextReportHeadedr returns header of text report -func (r ScanResult) FormatTextReportHeadedr() string { - serverInfo := r.ServerInfo() - var buf bytes.Buffer - for i := 0; i < len(serverInfo); i++ { - buf.WriteString("=") - } - return fmt.Sprintf("%s\n%s\n%s\t%s\n", - r.ServerInfo(), - buf.String(), - r.CveSummary(config.Conf.IgnoreUnscoredCves), - r.Packages.FormatUpdatablePacksSummary(), - ) -} - -// Confidence is a ranking how confident the CVE-ID was deteted correctly -// Score: 0 - 100 -type Confidence struct { - Score int - DetectionMethod string -} - -func (c Confidence) String() string { - return fmt.Sprintf("%d / %s", c.Score, c.DetectionMethod) -} - -const ( - // CpeNameMatchStr is a String representation of CpeNameMatch - CpeNameMatchStr = "CpeNameMatch" - - // YumUpdateSecurityMatchStr is a String representation of YumUpdateSecurityMatch - YumUpdateSecurityMatchStr = "YumUpdateSecurityMatch" - - // PkgAuditMatchStr is a String representation of PkgAuditMatch - PkgAuditMatchStr = "PkgAuditMatch" - - // OvalMatchStr is a String representation of OvalMatch - OvalMatchStr = "OvalMatch" - - // ChangelogExactMatchStr is a String representation of ChangelogExactMatch - ChangelogExactMatchStr = "ChangelogExactMatch" - - // ChangelogLenientMatchStr is a String representation of ChangelogLenientMatch - ChangelogLenientMatchStr = "ChangelogLenientMatch" - - // FailedToGetChangelog is a String representation of FailedToGetChangelog - FailedToGetChangelog = "FailedToGetChangelog" - - // FailedToFindVersionInChangelog is a String representation of FailedToFindVersionInChangelog - FailedToFindVersionInChangelog = "FailedToFindVersionInChangelog" -) - -var ( - // CpeNameMatch is a ranking how confident the CVE-ID was deteted correctly - CpeNameMatch = Confidence{100, CpeNameMatchStr} - - // YumUpdateSecurityMatch is a ranking how confident the CVE-ID was deteted correctly - YumUpdateSecurityMatch = Confidence{100, YumUpdateSecurityMatchStr} - - // PkgAuditMatch is a ranking how confident the CVE-ID was deteted correctly - PkgAuditMatch = Confidence{100, PkgAuditMatchStr} - - // OvalMatch is a ranking how confident the CVE-ID was deteted correctly - OvalMatch = Confidence{100, OvalMatchStr} - - // ChangelogExactMatch is a ranking how confident the CVE-ID was deteted correctly - ChangelogExactMatch = Confidence{95, ChangelogExactMatchStr} - - // ChangelogLenientMatch is a ranking how confident the CVE-ID was deteted correctly - ChangelogLenientMatch = Confidence{50, ChangelogLenientMatchStr} -) - -// VulnInfos is VulnInfo list, getter/setter, sortable methods. -type VulnInfos map[string]VulnInfo - -// Find elements that matches the function passed in argument -func (v VulnInfos) Find(f func(VulnInfo) bool) VulnInfos { - filtered := VulnInfos{} - for _, vv := range v { - if f(vv) { - filtered[vv.CveID] = vv - } - } - return filtered -} - -// FindScoredVulns return socred vulnerabilities -func (v VulnInfos) FindScoredVulns() VulnInfos { - return v.Find(func(vv VulnInfo) bool { - if 0 < vv.CveContents.MaxCvss2Score().Value.Score || - 0 < vv.CveContents.MaxCvss3Score().Value.Score { - return true - } - return false - }) -} - -// VulnInfo holds a vulnerability information and unsecure packages -type VulnInfo struct { - CveID string - Confidence Confidence - PackageNames []string - DistroAdvisories []DistroAdvisory // for Aamazon, RHEL, FreeBSD - CpeNames []string - CveContents CveContents -} - -// Cvss2CalcURL returns CVSS v2 caluclator's URL -func (v VulnInfo) Cvss2CalcURL() string { - return "https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=" + v.CveID -} - -// Cvss3CalcURL returns CVSS v3 caluclator's URL -func (v VulnInfo) Cvss3CalcURL() string { - return "https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator?name=" + v.CveID -} - -// TODO -// NilToEmpty set nil slice or map fields to empty to avoid null in JSON -// func (v *VulnInfo) NilToEmpty() { -// if v.CpeNames == nil { -// v.CpeNames = []string{} -// } -// if v.DistroAdvisories == nil { -// v.DistroAdvisories = []DistroAdvisory{} -// } -// if v.PackageNames == nil { -// v.PackageNames = []string{} -// } -// if v.CveContents == nil { -// v.CveContents = NewCveContents() -// } -// } - -// CveContentType is a source of CVE information -type CveContentType string - -// NewCveContentType create CveContentType -func NewCveContentType(name string) CveContentType { - switch name { - case "nvd": - return NVD - case "jvn": - return JVN - case "redhat", "centos": - return RedHat - case "ubuntu": - return Ubuntu - case "debian": - return Debian - default: - return Unknown - } -} - -const ( - // NVD is NVD - NVD CveContentType = "nvd" - - // JVN is JVN - JVN CveContentType = "jvn" - - // RedHat is RedHat - RedHat CveContentType = "redhat" - - // Debian is Debian - Debian CveContentType = "debian" - - // Ubuntu is Ubuntu - Ubuntu CveContentType = "ubuntu" - - // Unknown is Unknown - Unknown CveContentType = "unknown" -) - -// CveContentTypes has slide of CveContentType -type CveContentTypes []CveContentType - -// AllCveContetTypes has all of CveContentTypes -var AllCveContetTypes = CveContentTypes{NVD, JVN, RedHat, Debian, Ubuntu} - -// Except returns CveContentTypes except for given args -func (c CveContentTypes) Except(excepts ...CveContentType) (excepted CveContentTypes) { - for _, ctype := range c { - found := false - for _, except := range excepts { - if ctype == except { - found = true - break - } - } - if !found { - excepted = append(excepted, ctype) - } - } - return -} - -// CveContents has CveContent -type CveContents map[CveContentType]CveContent - -// NewCveContents create CveContents -func NewCveContents(conts ...CveContent) CveContents { - m := map[CveContentType]CveContent{} - for _, cont := range conts { - m[cont.Type] = cont - } - return m -} - -// CveContentStr has CveContentType and Value -type CveContentStr struct { - Type CveContentType - Value string -} - -// Except returns CveContents except given keys for enumeration -func (v CveContents) Except(exceptCtypes ...CveContentType) (values CveContents) { - for ctype, content := range v { - found := false - for _, exceptCtype := range exceptCtypes { - if ctype == exceptCtype { - found = true - break - } - } - if !found { - values[ctype] = content - } - } - return -} - -// CveContentCvss2 has CveContentType and Cvss2 -type CveContentCvss2 struct { - Type CveContentType - Value Cvss2 -} - -// Cvss2 has CVSS v2 -type Cvss2 struct { - Score float64 - Vector string - Severity string -} - -// Format CVSS Score and Vector -func (c Cvss2) Format() string { - return fmt.Sprintf("%3.1f/%s", c.Score, c.Vector) -} - -func cvss2ScoreToSeverity(score float64) string { - if 7.0 <= score { - return "HIGH" - } else if 4.0 <= score { - return "MEDIUM" - } - return "LOW" -} - -// Cvss2Scores returns CVSS V2 Scores -func (v CveContents) Cvss2Scores() (values []CveContentCvss2) { - order := []CveContentType{NVD, RedHat, JVN} - for _, ctype := range order { - if cont, found := v[ctype]; found && 0 < cont.Cvss2Score { - // https://nvd.nist.gov/vuln-metrics/cvss - sev := cont.Severity - if ctype == NVD { - sev = cvss2ScoreToSeverity(cont.Cvss2Score) - } - values = append(values, CveContentCvss2{ - Type: ctype, - Value: Cvss2{ - Score: cont.Cvss2Score, - Vector: cont.Cvss2Vector, - Severity: sev, - }, - }) - } - } - return -} - -// MaxCvss2Score returns Max CVSS V2 Score -func (v CveContents) MaxCvss2Score() CveContentCvss2 { - //TODO Severity Ubuntu, Debian... - order := []CveContentType{NVD, RedHat, JVN} - max := 0.0 - value := CveContentCvss2{ - Type: Unknown, - Value: Cvss2{}, - } - for _, ctype := range order { - if cont, found := v[ctype]; found && max < cont.Cvss2Score { - // https://nvd.nist.gov/vuln-metrics/cvss - sev := cont.Severity - if ctype == NVD { - sev = cvss2ScoreToSeverity(cont.Cvss2Score) - } - value = CveContentCvss2{ - Type: ctype, - Value: Cvss2{ - Score: cont.Cvss2Score, - Vector: cont.Cvss2Vector, - Severity: sev, - }, - } - max = cont.Cvss2Score - } - } - return value -} - -// CveContentCvss3 has CveContentType and Cvss3 -type CveContentCvss3 struct { - Type CveContentType - Value Cvss3 -} - -// Cvss3 has CVSS v3 Score, Vector and Severity -type Cvss3 struct { - Score float64 - Vector string - Severity string -} - -// Format CVSS Score and Vector -func (c Cvss3) Format() string { - return fmt.Sprintf("%3.1f/CVSS:3.0/%s", c.Score, c.Vector) -} - -func cvss3ScoreToSeverity(score float64) string { - if 9.0 <= score { - return "CRITICAL" - } else if 7.0 <= score { - return "HIGH" - } else if 4.0 <= score { - return "MEDIUM" - } - return "LOW" -} - -// Cvss3Scores returns CVSS V3 Score -func (v CveContents) Cvss3Scores() (values []CveContentCvss3) { - //TODO Severity Ubuntu, Debian... - order := []CveContentType{RedHat} - for _, ctype := range order { - if cont, found := v[ctype]; found && 0 < cont.Cvss3Score { - // https://nvd.nist.gov/vuln-metrics/cvss - sev := cont.Severity - if ctype == NVD { - sev = cvss3ScoreToSeverity(cont.Cvss2Score) - } - values = append(values, CveContentCvss3{ - Type: ctype, - Value: Cvss3{ - Score: cont.Cvss3Score, - Vector: cont.Cvss3Vector, - Severity: sev, - }, - }) - } - } - return -} - -// MaxCvss3Score returns Max CVSS V3 Score -func (v CveContents) MaxCvss3Score() CveContentCvss3 { - //TODO Severity Ubuntu, Debian... - order := []CveContentType{RedHat} - max := 0.0 - value := CveContentCvss3{ - Type: Unknown, - Value: Cvss3{}, - } - for _, ctype := range order { - if cont, found := v[ctype]; found && max < cont.Cvss3Score { - // https://nvd.nist.gov/vuln-metrics/cvss - sev := cont.Severity - if ctype == NVD { - sev = cvss3ScoreToSeverity(cont.Cvss2Score) - } - value = CveContentCvss3{ - Type: ctype, - Value: Cvss3{ - Score: cont.Cvss3Score, - Vector: cont.Cvss3Vector, - Severity: sev, - }, - } - max = cont.Cvss3Score - } - } - return value -} - -// FormatMaxCvssScore returns Max CVSS Score -func (v CveContents) FormatMaxCvssScore() string { - v2Max := v.MaxCvss2Score() - v3Max := v.MaxCvss3Score() - if v2Max.Value.Score <= v3Max.Value.Score { - return fmt.Sprintf("%3.1f %s (%s)", - v3Max.Value.Score, - strings.ToUpper(v3Max.Value.Severity), - v3Max.Type) - } - return fmt.Sprintf("%3.1f %s (%s)", - v2Max.Value.Score, - strings.ToUpper(v2Max.Value.Severity), - v2Max.Type) -} - -// Titles returns tilte (TUI) -func (v CveContents) Titles(lang, myFamily string) (values []CveContentStr) { - if lang == "ja" { - if cont, found := v[JVN]; found && 0 < len(cont.Title) { - values = append(values, CveContentStr{JVN, cont.Title}) - } - } - - order := CveContentTypes{NVD, NewCveContentType(myFamily)} - order = append(order, AllCveContetTypes.Except(append(order, JVN)...)...) - for _, ctype := range order { - // Only JVN has meaningful title. so return first 100 char of summary - if cont, found := v[ctype]; found && 0 < len(cont.Summary) { - summary := strings.Replace(cont.Summary, "\n", " ", -1) - index := 75 - if len(summary) < index { - index = len(summary) - } - values = append(values, CveContentStr{ - Type: ctype, - Value: summary[0:index] + "...", - }) - } - } - - if len(values) == 0 { - values = []CveContentStr{{ - Type: Unknown, - Value: "-", - }} - } - return -} - -// Summaries returns summaries -func (v CveContents) Summaries(lang, myFamily string) (values []CveContentStr) { - if lang == "ja" { - if cont, found := v[JVN]; found && 0 < len(cont.Summary) { - summary := cont.Title - summary += "\n" + strings.Replace( - strings.Replace(cont.Summary, "\n", " ", -1), "\r", " ", -1) - values = append(values, CveContentStr{JVN, summary}) - } - } - - order := CveContentTypes{NVD, NewCveContentType(myFamily)} - order = append(order, AllCveContetTypes.Except(append(order, JVN)...)...) - for _, ctype := range order { - if cont, found := v[ctype]; found && 0 < len(cont.Summary) { - summary := strings.Replace(cont.Summary, "\n", " ", -1) - values = append(values, CveContentStr{ - Type: ctype, - Value: summary, - }) - } - } - - if len(values) == 0 { - values = []CveContentStr{{ - Type: Unknown, - Value: "-", - }} - } - return -} - -// SourceLinks returns link of source -func (v CveContents) SourceLinks(lang, myFamily, cveID string) (values []CveContentStr) { - if lang == "ja" { - if cont, found := v[JVN]; found && !cont.Empty() { - values = append(values, CveContentStr{JVN, cont.SourceLink}) - } - } - - order := CveContentTypes{NVD, NewCveContentType(myFamily)} - for _, ctype := range order { - if cont, found := v[ctype]; found { - values = append(values, CveContentStr{ctype, cont.SourceLink}) - } - } - - if len(values) == 0 { - return []CveContentStr{{ - Type: NVD, - Value: "https://nvd.nist.gov/vuln/detail/" + cveID, - }} - } - return values -} - -// VendorLink returns link of source -func (v CveContents) VendorLink(myFamily string) CveContentStr { - ctype := NewCveContentType(myFamily) - if cont, ok := v[ctype]; ok { - return CveContentStr{ctype, cont.SourceLink} - } - return CveContentStr{ctype, ""} -} - -// Severities returns Severities -// func (v CveContents) Severities(myFamily string) (values []CveContentValue) { -// order := CveContentTypes{NVD, NewCveContentType(myFamily)} -// order = append(order, AllCveContetTypes.Except(append(order)...)...) - -// for _, ctype := range order { -// if cont, found := v[ctype]; found && 0 < len(cont.Severity) { -// values = append(values, CveContentValue{ -// Type: ctype, -// Value: cont.Severity, -// }) -// } -// } -// return -// } - -// CveContentCpes has CveContentType and Value -type CveContentCpes struct { - Type CveContentType - Value []Cpe -} - -// Cpes returns affected CPEs of this Vulnerability -func (v CveContents) Cpes(myFamily string) (values []CveContentCpes) { - order := CveContentTypes{NewCveContentType(myFamily)} - order = append(order, AllCveContetTypes.Except(append(order)...)...) - - for _, ctype := range order { - if cont, found := v[ctype]; found && 0 < len(cont.Cpes) { - values = append(values, CveContentCpes{ - Type: ctype, - Value: cont.Cpes, - }) - } - } - return -} - -// CveContentRefs has CveContentType and Cpes -type CveContentRefs struct { - Type CveContentType - Value []Reference -} - -// References returns References -func (v CveContents) References(myFamily string) (values []CveContentRefs) { - order := CveContentTypes{NewCveContentType(myFamily)} - order = append(order, AllCveContetTypes.Except(append(order)...)...) - - for _, ctype := range order { - if cont, found := v[ctype]; found && 0 < len(cont.References) { - values = append(values, CveContentRefs{ - Type: ctype, - Value: cont.References, - }) - } - } - return -} - -// CweIDs returns related CweIDs of the vulnerability -func (v CveContents) CweIDs(myFamily string) (values []CveContentStr) { - order := CveContentTypes{NewCveContentType(myFamily)} - order = append(order, AllCveContetTypes.Except(append(order)...)...) - - for _, ctype := range order { - if cont, found := v[ctype]; found && 0 < len(cont.CweID) { - // RedHat's OVAL sometimes contains multiple CWE-IDs separated by spaces - for _, cweID := range strings.Fields(cont.CweID) { - values = append(values, CveContentStr{ - Type: ctype, - Value: cweID, - }) - } - } - } - return -} - -// CveContent has abstraction of various vulnerability information -type CveContent struct { - Type CveContentType - CveID string - Title string - Summary string - Severity string - Cvss2Score float64 - Cvss2Vector string - Cvss3Score float64 - Cvss3Vector string - SourceLink string - Cpes []Cpe - References References - CweID string - Published time.Time - LastModified time.Time -} - -// Empty checks the content is empty -func (c CveContent) Empty() bool { - return c.Summary == "" -} - -// Cpe is Common Platform Enumeration -type Cpe struct { - CpeName string -} - -// References is a slice of Reference -type References []Reference - -// Find elements that matches the function passed in argument -func (r References) Find(f func(r Reference) bool) (refs []Reference) { - for _, rr := range r { - refs = append(refs, rr) - } - return -} - -// Reference has a related link of the CVE -type Reference struct { - Source string - Link string - RefID string -} - -// Packages is Map of Package -// { "package-name": Package } -type Packages map[string]Package - -// NewPackages create Packages -func NewPackages(packs ...Package) Packages { - m := Packages{} - for _, pack := range packs { - m[pack.Name] = pack - } - return m -} - -// MergeNewVersion merges candidate version information to the receiver struct -func (ps Packages) MergeNewVersion(as Packages) { - for _, a := range as { - if pack, ok := ps[a.Name]; ok { - pack.NewVersion = a.NewVersion - pack.NewRelease = a.NewRelease - ps[a.Name] = pack - } - } -} - -// Merge returns merged map (immutable) -func (ps Packages) Merge(other Packages) Packages { - merged := map[string]Package{} - for k, v := range ps { - merged[k] = v - } - for k, v := range other { - merged[k] = v - } - return merged -} - -// FormatVersionsFromTo returns updatable packages -func (ps Packages) FormatVersionsFromTo() string { - ss := []string{} - for _, pack := range ps { - ss = append(ss, pack.FormatVersionFromTo()) - } - return strings.Join(ss, "\n") -} - -// FormatUpdatablePacksSummary returns a summary of updatable packages -func (ps Packages) FormatUpdatablePacksSummary() string { - nUpdatable := 0 - for _, p := range ps { - if p.NewVersion != "" { - nUpdatable++ - } - } - return fmt.Sprintf("%d updatable packages", nUpdatable) -} - -// Package has installed packages. -type Package struct { - Name string - Version string - Release string - NewVersion string - NewRelease string - Repository string - Changelog Changelog - NotFixedYet bool // Ubuntu OVAL Only -} - -// FormatVer returns package name-version-release -func (p Package) FormatVer() string { - str := p.Name - if 0 < len(p.Version) { - str = fmt.Sprintf("%s-%s", str, p.Version) - } - if 0 < len(p.Release) { - str = fmt.Sprintf("%s-%s", str, p.Release) - } - return str -} - -// FormatNewVer returns package name-version-release -func (p Package) FormatNewVer() string { - str := p.Name - if 0 < len(p.NewVersion) { - str = fmt.Sprintf("%s-%s", str, p.NewVersion) - } - if 0 < len(p.NewRelease) { - str = fmt.Sprintf("%s-%s", str, p.NewRelease) - } - return str -} - -// FormatVersionFromTo formats installed and new package version -func (p Package) FormatVersionFromTo() string { - return fmt.Sprintf("%s -> %s", p.FormatVer(), p.FormatNewVer()) -} - -// Changelog has contents of changelog and how to get it. -// Method: modesl.detectionMethodStr -type Changelog struct { - Contents string - Method string -} - -// DistroAdvisory has Amazon Linux, RHEL, FreeBSD Security Advisory information. -type DistroAdvisory struct { - AdvisoryID string - Severity string - Issued time.Time - Updated time.Time -} - -// Container has Container information -type Container struct { - ContainerID string - Name string - Image string - Type string -} - -// Platform has platform information -type Platform struct { - Name string // aws or azure or gcp or other... - InstanceID string -} diff --git a/models/models_test.go b/models/models_test.go index 7229f0ee..aee32778 100644 --- a/models/models_test.go +++ b/models/models_test.go @@ -16,45 +16,3 @@ along with this program. If not, see . */ package models - -import ( - "reflect" - "testing" - - "github.com/k0kubun/pp" -) - -func TestMergeNewVersion(t *testing.T) { - var test = struct { - a Packages - b Packages - expected Packages - }{ - Packages{ - "hoge": { - Name: "hoge", - }, - }, - Packages{ - "hoge": { - Name: "hoge", - NewVersion: "1.0.0", - NewRelease: "release1", - }, - }, - Packages{ - "hoge": { - Name: "hoge", - NewVersion: "1.0.0", - NewRelease: "release1", - }, - }, - } - - test.a.MergeNewVersion(test.b) - if !reflect.DeepEqual(test.a, test.expected) { - e := pp.Sprintf("%v", test.a) - a := pp.Sprintf("%v", test.expected) - t.Errorf("expected %s, actual %s", e, a) - } -} diff --git a/models/packages.go b/models/packages.go new file mode 100644 index 00000000..d923dfc5 --- /dev/null +++ b/models/packages.go @@ -0,0 +1,127 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package models + +import ( + "fmt" + "strings" +) + +// Packages is Map of Package +// { "package-name": Package } +type Packages map[string]Package + +// NewPackages create Packages +func NewPackages(packs ...Package) Packages { + m := Packages{} + for _, pack := range packs { + m[pack.Name] = pack + } + return m +} + +// MergeNewVersion merges candidate version information to the receiver struct +func (ps Packages) MergeNewVersion(as Packages) { + for _, a := range as { + if pack, ok := ps[a.Name]; ok { + pack.NewVersion = a.NewVersion + pack.NewRelease = a.NewRelease + ps[a.Name] = pack + } + } +} + +// Merge returns merged map (immutable) +func (ps Packages) Merge(other Packages) Packages { + merged := map[string]Package{} + for k, v := range ps { + merged[k] = v + } + for k, v := range other { + merged[k] = v + } + return merged +} + +// FormatVersionsFromTo returns updatable packages +func (ps Packages) FormatVersionsFromTo() string { + ss := []string{} + for _, pack := range ps { + ss = append(ss, pack.FormatVersionFromTo()) + } + return strings.Join(ss, "\n") +} + +// FormatUpdatablePacksSummary returns a summary of updatable packages +func (ps Packages) FormatUpdatablePacksSummary() string { + nUpdatable := 0 + for _, p := range ps { + if p.NewVersion != "" { + nUpdatable++ + } + } + return fmt.Sprintf("%d updatable packages", nUpdatable) +} + +// Package has installed packages. +type Package struct { + Name string + Version string + Release string + NewVersion string + NewRelease string + Repository string + Changelog Changelog + NotFixedYet bool // Ubuntu OVAL Only +} + +// FormatVer returns package name-version-release +func (p Package) FormatVer() string { + str := p.Name + if 0 < len(p.Version) { + str = fmt.Sprintf("%s-%s", str, p.Version) + } + if 0 < len(p.Release) { + str = fmt.Sprintf("%s-%s", str, p.Release) + } + return str +} + +// FormatNewVer returns package name-version-release +func (p Package) FormatNewVer() string { + str := p.Name + if 0 < len(p.NewVersion) { + str = fmt.Sprintf("%s-%s", str, p.NewVersion) + } + if 0 < len(p.NewRelease) { + str = fmt.Sprintf("%s-%s", str, p.NewRelease) + } + return str +} + +// FormatVersionFromTo formats installed and new package version +func (p Package) FormatVersionFromTo() string { + return fmt.Sprintf("%s -> %s", p.FormatVer(), p.FormatNewVer()) +} + +// Changelog has contents of changelog and how to get it. +// Method: modesl.detectionMethodStr +type Changelog struct { + Contents string + Method string +} diff --git a/models/packages_test.go b/models/packages_test.go new file mode 100644 index 00000000..321b07db --- /dev/null +++ b/models/packages_test.go @@ -0,0 +1,59 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package models + +import ( + "reflect" + "testing" + + "github.com/k0kubun/pp" +) + +func TestMergeNewVersion(t *testing.T) { + var test = struct { + a Packages + b Packages + expected Packages + }{ + Packages{ + "hoge": { + Name: "hoge", + }, + }, + Packages{ + "hoge": { + Name: "hoge", + NewVersion: "1.0.0", + NewRelease: "release1", + }, + }, + Packages{ + "hoge": { + Name: "hoge", + NewVersion: "1.0.0", + NewRelease: "release1", + }, + }, + } + + test.a.MergeNewVersion(test.b) + if !reflect.DeepEqual(test.a, test.expected) { + e := pp.Sprintf("%v", test.a) + a := pp.Sprintf("%v", test.expected) + t.Errorf("expected %s, actual %s", e, a) + } +} diff --git a/models/scanresults.go b/models/scanresults.go new file mode 100644 index 00000000..a276ef25 --- /dev/null +++ b/models/scanresults.go @@ -0,0 +1,297 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package models + +import ( + "bytes" + "fmt" + "strings" + "time" + + "github.com/future-architect/vuls/config" + cvedict "github.com/kotakanbe/go-cve-dictionary/models" +) + +// ScanResults is slice of ScanResult. +type ScanResults []ScanResult + +//TODO +// // Len implement Sort Interface +// func (s ScanResults) Len() int { +// return len(s) +// } + +// // Swap implement Sort Interface +// func (s ScanResults) Swap(i, j int) { +// s[i], s[j] = s[j], s[i] +// } + +// // Less implement Sort Interface +// func (s ScanResults) Less(i, j int) bool { +// if s[i].ServerName == s[j].ServerName { +// return s[i].Container.ContainerID < s[i].Container.ContainerID +// } +// return s[i].ServerName < s[j].ServerName +// } + +// ScanResult has the result of scanned CVE information. +type ScanResult struct { + ScannedAt time.Time + JSONVersion string + Lang string + ServerName string // TOML Section key + Family string + Release string + Container Container + Platform Platform + + // Scanned Vulns by SSH scan + CPE + OVAL + ScannedCves VulnInfos + + Packages Packages + Errors []string + Optional [][]interface{} +} + +// ConvertNvdToModel convert NVD to CveContent +func (r ScanResult) ConvertNvdToModel(cveID string, nvd cvedict.Nvd) *CveContent { + var cpes []Cpe + for _, c := range nvd.Cpes { + cpes = append(cpes, Cpe{CpeName: c.CpeName}) + } + + var refs []Reference + for _, r := range nvd.References { + refs = append(refs, Reference{ + Link: r.Link, + Source: r.Source, + }) + } + + validVec := true + for _, v := range []string{ + nvd.AccessVector, + nvd.AccessComplexity, + nvd.Authentication, + nvd.ConfidentialityImpact, + nvd.IntegrityImpact, + nvd.AvailabilityImpact, + } { + if len(v) == 0 { + validVec = false + } + } + + vector := "" + if validVec { + vector = fmt.Sprintf("AV:%s/AC:%s/Au:%s/C:%s/I:%s/A:%s", + string(nvd.AccessVector[0]), + string(nvd.AccessComplexity[0]), + string(nvd.Authentication[0]), + string(nvd.ConfidentialityImpact[0]), + string(nvd.IntegrityImpact[0]), + string(nvd.AvailabilityImpact[0])) + } + + //TODO CVSSv3 + return &CveContent{ + Type: NVD, + CveID: cveID, + Summary: nvd.Summary, + Cvss2Score: nvd.Score, + Cvss2Vector: vector, + Severity: "", // severity is not contained in NVD + SourceLink: "https://nvd.nist.gov/vuln/detail/" + cveID, + Cpes: cpes, + CweID: nvd.CweID, + References: refs, + Published: nvd.PublishedDate, + LastModified: nvd.LastModifiedDate, + } +} + +// ConvertJvnToModel convert JVN to CveContent +func (r ScanResult) ConvertJvnToModel(cveID string, jvn cvedict.Jvn) *CveContent { + var cpes []Cpe + for _, c := range jvn.Cpes { + cpes = append(cpes, Cpe{CpeName: c.CpeName}) + } + + refs := []Reference{} + for _, r := range jvn.References { + refs = append(refs, Reference{ + Link: r.Link, + Source: r.Source, + }) + } + + vector := strings.TrimSuffix(strings.TrimPrefix(jvn.Vector, "("), ")") + return &CveContent{ + Type: JVN, + CveID: cveID, + Title: jvn.Title, + Summary: jvn.Summary, + Severity: jvn.Severity, + Cvss2Score: jvn.Score, + Cvss2Vector: vector, + SourceLink: jvn.JvnLink, + Cpes: cpes, + References: refs, + Published: jvn.PublishedDate, + LastModified: jvn.LastModifiedDate, + } +} + +// FilterByCvssOver is filter function. +func (r ScanResult) FilterByCvssOver(over float64) ScanResult { + // TODO: Set correct default value + if over == 0 { + over = -1.1 + } + + // TODO: Filter by ignore cves??? + filtered := r.ScannedCves.Find(func(v VulnInfo) bool { + //TODO in the case of only oval, no cvecontents + values := v.CveContents.Cvss2Scores() + for _, vals := range values { + score := vals.Value.Score + if over <= score { + return true + } + } + return false + }) + + copiedScanResult := r + copiedScanResult.ScannedCves = filtered + return copiedScanResult +} + +// ReportFileName returns the filename on localhost without extention +func (r ScanResult) ReportFileName() (name string) { + if len(r.Container.ContainerID) == 0 { + return fmt.Sprintf("%s", r.ServerName) + } + return fmt.Sprintf("%s@%s", r.Container.Name, r.ServerName) +} + +// ReportKeyName returns the name of key on S3, Azure-Blob without extention +func (r ScanResult) ReportKeyName() (name string) { + timestr := r.ScannedAt.Format(time.RFC3339) + if len(r.Container.ContainerID) == 0 { + return fmt.Sprintf("%s/%s", timestr, r.ServerName) + } + return fmt.Sprintf("%s/%s@%s", timestr, r.Container.Name, r.ServerName) +} + +// ServerInfo returns server name one line +func (r ScanResult) ServerInfo() string { + if len(r.Container.ContainerID) == 0 { + return fmt.Sprintf("%s (%s%s)", + r.ServerName, r.Family, r.Release) + } + return fmt.Sprintf( + "%s / %s (%s%s) on %s", + r.Container.Name, + r.Container.ContainerID, + r.Family, + r.Release, + r.ServerName, + ) +} + +// ServerInfoTui returns server infromation for TUI sidebar +func (r ScanResult) ServerInfoTui() string { + if len(r.Container.ContainerID) == 0 { + return fmt.Sprintf("%s (%s%s)", + r.ServerName, r.Family, r.Release) + } + return fmt.Sprintf( + "|-- %s (%s%s)", + r.Container.Name, + r.Family, + r.Release, + // r.Container.ContainerID, + ) +} + +// FormatServerName returns server and container name +func (r ScanResult) FormatServerName() string { + if len(r.Container.ContainerID) == 0 { + return r.ServerName + } + return fmt.Sprintf("%s@%s", + r.Container.Name, r.ServerName) +} + +// CveSummary summarize the number of CVEs group by CVSSv2 Severity +func (r ScanResult) CveSummary(ignoreUnscoreCves bool) string { + var high, medium, low, unknown int + for _, vInfo := range r.ScannedCves { + score := vInfo.CveContents.MaxCvss2Score().Value.Score + if score < 0.1 { + score = vInfo.CveContents.MaxCvss3Score().Value.Score + } + switch { + case 7.0 <= score: + high++ + case 4.0 <= score: + medium++ + case 0 < score: + low++ + default: + unknown++ + } + } + + if ignoreUnscoreCves { + return fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d)", + high+medium+low, high, medium, low) + } + return fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d ?:%d)", + high+medium+low+unknown, high, medium, low, unknown) +} + +// FormatTextReportHeadedr returns header of text report +func (r ScanResult) FormatTextReportHeadedr() string { + serverInfo := r.ServerInfo() + var buf bytes.Buffer + for i := 0; i < len(serverInfo); i++ { + buf.WriteString("=") + } + return fmt.Sprintf("%s\n%s\n%s\t%s\n", + r.ServerInfo(), + buf.String(), + r.CveSummary(config.Conf.IgnoreUnscoredCves), + r.Packages.FormatUpdatablePacksSummary(), + ) +} + +// Container has Container information +type Container struct { + ContainerID string + Name string + Image string + Type string +} + +// Platform has platform information +type Platform struct { + Name string // aws or azure or gcp or other... + InstanceID string +} diff --git a/models/scanresults_test.go b/models/scanresults_test.go new file mode 100644 index 00000000..fced7012 --- /dev/null +++ b/models/scanresults_test.go @@ -0,0 +1,17 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package models diff --git a/models/vulninfos.go b/models/vulninfos.go new file mode 100644 index 00000000..a1a03540 --- /dev/null +++ b/models/vulninfos.go @@ -0,0 +1,150 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package models + +import ( + "fmt" + "time" +) + +// VulnInfos is VulnInfo list, getter/setter, sortable methods. +type VulnInfos map[string]VulnInfo + +// Find elements that matches the function passed in argument +func (v VulnInfos) Find(f func(VulnInfo) bool) VulnInfos { + filtered := VulnInfos{} + for _, vv := range v { + if f(vv) { + filtered[vv.CveID] = vv + } + } + return filtered +} + +// FindScoredVulns return socred vulnerabilities +func (v VulnInfos) FindScoredVulns() VulnInfos { + return v.Find(func(vv VulnInfo) bool { + if 0 < vv.CveContents.MaxCvss2Score().Value.Score || + 0 < vv.CveContents.MaxCvss3Score().Value.Score { + return true + } + return false + }) +} + +// VulnInfo holds a vulnerability information and unsecure packages +type VulnInfo struct { + CveID string + Confidence Confidence + PackageNames []string + DistroAdvisories []DistroAdvisory // for Aamazon, RHEL, FreeBSD + CpeNames []string + CveContents CveContents +} + +// Cvss2CalcURL returns CVSS v2 caluclator's URL +func (v VulnInfo) Cvss2CalcURL() string { + return "https://nvd.nist.gov/vuln-metrics/cvss/v2-calculator?name=" + v.CveID +} + +// Cvss3CalcURL returns CVSS v3 caluclator's URL +func (v VulnInfo) Cvss3CalcURL() string { + return "https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator?name=" + v.CveID +} + +// TODO +// NilToEmpty set nil slice or map fields to empty to avoid null in JSON +// func (v *VulnInfo) NilToEmpty() { +// if v.CpeNames == nil { +// v.CpeNames = []string{} +// } +// if v.DistroAdvisories == nil { +// v.DistroAdvisories = []DistroAdvisory{} +// } +// if v.PackageNames == nil { +// v.PackageNames = []string{} +// } +// if v.CveContents == nil { +// v.CveContents = NewCveContents() +// } +// } + +// DistroAdvisory has Amazon Linux, RHEL, FreeBSD Security Advisory information. +type DistroAdvisory struct { + AdvisoryID string + Severity string + Issued time.Time + Updated time.Time +} + +// Confidence is a ranking how confident the CVE-ID was deteted correctly +// Score: 0 - 100 +type Confidence struct { + Score int + DetectionMethod string +} + +func (c Confidence) String() string { + return fmt.Sprintf("%d / %s", c.Score, c.DetectionMethod) +} + +const ( + // CpeNameMatchStr is a String representation of CpeNameMatch + CpeNameMatchStr = "CpeNameMatch" + + // YumUpdateSecurityMatchStr is a String representation of YumUpdateSecurityMatch + YumUpdateSecurityMatchStr = "YumUpdateSecurityMatch" + + // PkgAuditMatchStr is a String representation of PkgAuditMatch + PkgAuditMatchStr = "PkgAuditMatch" + + // OvalMatchStr is a String representation of OvalMatch + OvalMatchStr = "OvalMatch" + + // ChangelogExactMatchStr is a String representation of ChangelogExactMatch + ChangelogExactMatchStr = "ChangelogExactMatch" + + // ChangelogLenientMatchStr is a String representation of ChangelogLenientMatch + ChangelogLenientMatchStr = "ChangelogLenientMatch" + + // FailedToGetChangelog is a String representation of FailedToGetChangelog + FailedToGetChangelog = "FailedToGetChangelog" + + // FailedToFindVersionInChangelog is a String representation of FailedToFindVersionInChangelog + FailedToFindVersionInChangelog = "FailedToFindVersionInChangelog" +) + +var ( + // CpeNameMatch is a ranking how confident the CVE-ID was deteted correctly + CpeNameMatch = Confidence{100, CpeNameMatchStr} + + // YumUpdateSecurityMatch is a ranking how confident the CVE-ID was deteted correctly + YumUpdateSecurityMatch = Confidence{100, YumUpdateSecurityMatchStr} + + // PkgAuditMatch is a ranking how confident the CVE-ID was deteted correctly + PkgAuditMatch = Confidence{100, PkgAuditMatchStr} + + // OvalMatch is a ranking how confident the CVE-ID was deteted correctly + OvalMatch = Confidence{100, OvalMatchStr} + + // ChangelogExactMatch is a ranking how confident the CVE-ID was deteted correctly + ChangelogExactMatch = Confidence{95, ChangelogExactMatchStr} + + // ChangelogLenientMatch is a ranking how confident the CVE-ID was deteted correctly + ChangelogLenientMatch = Confidence{50, ChangelogLenientMatchStr} +) diff --git a/models/vulninfos_test.go b/models/vulninfos_test.go new file mode 100644 index 00000000..fced7012 --- /dev/null +++ b/models/vulninfos_test.go @@ -0,0 +1,17 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ +package models diff --git a/cveapi/cve_client.go b/report/cve_client.go similarity index 99% rename from cveapi/cve_client.go rename to report/cve_client.go index 08a73164..fad79470 100644 --- a/cveapi/cve_client.go +++ b/report/cve_client.go @@ -15,7 +15,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -package cveapi +package report import ( "encoding/json" diff --git a/report/report.go b/report/report.go new file mode 100644 index 00000000..0d77f766 --- /dev/null +++ b/report/report.go @@ -0,0 +1,204 @@ +/* Vuls - Vulnerability Scanner +Copyright (C) 2016 Future Architect, Inc. Japan. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +package report + +import ( + "fmt" + "os" + + c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/oval" + "github.com/future-architect/vuls/util" + "github.com/k0kubun/pp" +) + +// FillCveInfos fills CVE Detailed Information +func FillCveInfos(rs []models.ScanResult, dir string) ([]models.ScanResult, error) { + var filled []models.ScanResult + for _, r := range rs { + if c.Conf.RefreshCve || needToRefreshCve(r) { + if err := fillCveInfo(&r); err != nil { + return nil, err + } + r.Lang = c.Conf.Lang + if err := overwriteJSONFile(dir, r); err != nil { + return nil, fmt.Errorf("Failed to write JSON: %s", err) + } + filled = append(filled, r) + } else { + util.Log.Debugf("No need to refresh") + filled = append(filled, r) + } + } + + if c.Conf.Diff { + previous, err := loadPrevious(filled) + if err != nil { + return nil, err + } + + diff, err := diff(filled, previous) + if err != nil { + return nil, err + } + filled = []models.ScanResult{} + for _, r := range diff { + if err := fillCveDetail(&r); err != nil { + return nil, err + } + filled = append(filled, r) + } + } + + for _, r := range filled { + pp.Printf("filled: %d\n", len(r.ScannedCves)) + } + + filtered := []models.ScanResult{} + for _, r := range filled { + filtered = append(filtered, r.FilterByCvssOver(c.Conf.CvssScoreOver)) + } + + for _, r := range filtered { + pp.Printf("filtered: %d\n", len(r.ScannedCves)) + } + + // TODO Sort + return filtered, nil +} + +func fillCveInfo(r *models.ScanResult) error { + util.Log.Debugf("need to refresh") + if c.Conf.CveDBType == "sqlite3" && c.Conf.CveDBURL == "" { + if _, err := os.Stat(c.Conf.CveDBPath); os.IsNotExist(err) { + return fmt.Errorf("SQLite3 DB(CVE-Dictionary) is not exist: %s", + c.Conf.CveDBPath) + } + } + + if err := fillCveInfoFromOvalDB(r); err != nil { + return fmt.Errorf("Failed to fill OVAL information: %s", err) + } + + if err := fillCveInfoFromCveDB(r); err != nil { + return fmt.Errorf("Failed to fill CVE information: %s", err) + } + return nil +} + +// fillCveDetail fetches NVD, JVN from CVE Database, and then set to fields. +func fillCveDetail(r *models.ScanResult) error { + var cveIDs []string + for _, v := range r.ScannedCves { + cveIDs = append(cveIDs, v.CveID) + } + + ds, err := CveClient.FetchCveDetails(cveIDs) + if err != nil { + return err + } + for _, d := range ds { + nvd := r.ConvertNvdToModel(d.CveID, d.Nvd) + jvn := r.ConvertJvnToModel(d.CveID, d.Jvn) + for cveID, vinfo := range r.ScannedCves { + if vinfo.CveID == d.CveID { + if vinfo.CveContents == nil { + vinfo.CveContents = models.CveContents{} + } + for _, con := range []models.CveContent{*nvd, *jvn} { + if !con.Empty() { + vinfo.CveContents[con.Type] = con + } + } + r.ScannedCves[cveID] = vinfo + break + } + } + } + //TODO Remove + // sort.Slice(r.ScannedCves, func(i, j int) bool { + // if r.ScannedCves[j].CveContents.CvssV2Score() == r.ScannedCves[i].CveContents.CvssV2Score() { + // return r.ScannedCves[j].CveContents.CvssV2Score() < r.ScannedCves[i].CveContents.CvssV2Score() + // } + // return r.ScannedCves[j].CveContents.CvssV2Score() < r.ScannedCves[i].CveContents.CvssV2Score() + // }) + return nil +} + +func fillCveInfoFromCveDB(r *models.ScanResult) error { + sInfo := c.Conf.Servers[r.ServerName] + if err := fillVulnByCpeNames(sInfo.CpeNames, r.ScannedCves); err != nil { + return err + } + if err := fillCveDetail(r); err != nil { + return err + } + return nil +} + +func fillCveInfoFromOvalDB(r *models.ScanResult) error { + var ovalClient oval.Client + switch r.Family { + case "debian": + ovalClient = oval.NewDebian() + case "ubuntu": + ovalClient = oval.NewUbuntu() + case "rhel": + ovalClient = oval.NewRedhat() + case "centos": + ovalClient = oval.NewCentOS() + case "amazon", "oraclelinux", "Raspbian", "FreeBSD": + //TODO implement OracleLinux + return nil + default: + return fmt.Errorf("Oval %s is not implemented yet", r.Family) + } + if err := ovalClient.FillCveInfoFromOvalDB(r); err != nil { + return err + } + return nil +} + +func fillVulnByCpeNames(cpeNames []string, scannedVulns models.VulnInfos) error { + for _, name := range cpeNames { + details, err := CveClient.FetchCveDetailsByCpeName(name) + if err != nil { + return err + } + for _, detail := range details { + if val, ok := scannedVulns[detail.CveID]; ok { + names := val.CpeNames + names = util.AppendIfMissing(names, name) + val.CpeNames = names + val.Confidence = models.CpeNameMatch + scannedVulns[detail.CveID] = val + } else { + v := models.VulnInfo{ + CveID: detail.CveID, + CpeNames: []string{name}, + Confidence: models.CpeNameMatch, + } + //TODO + // v.NilToEmpty() + scannedVulns[detail.CveID] = v + } + } + } + return nil +} diff --git a/report/report_test.go b/report/report_test.go new file mode 100644 index 00000000..80c499fb --- /dev/null +++ b/report/report_test.go @@ -0,0 +1 @@ +package report diff --git a/report/util.go b/report/util.go index 405d7fb1..9e4eb230 100644 --- a/report/util.go +++ b/report/util.go @@ -19,11 +19,19 @@ package report import ( "bytes" + "encoding/json" "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "sort" "strings" + "time" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" "github.com/gosuri/uitable" ) @@ -516,3 +524,256 @@ func formatOneChangelog(p models.Package) string { buf = append(buf, packVer, delim.String(), clog) return strings.Join(buf, "\n") } + +func needToRefreshCve(r models.ScanResult) bool { + if r.Lang != config.Conf.Lang { + return true + } + + for _, cve := range r.ScannedCves { + if 0 < len(cve.CveContents) { + return false + } + } + return true +} + +func overwriteJSONFile(dir string, r models.ScanResult) error { + before := config.Conf.FormatJSON + beforeDiff := config.Conf.Diff + config.Conf.FormatJSON = true + config.Conf.Diff = false + w := LocalFileWriter{CurrentDir: dir} + if err := w.Write(r); err != nil { + return fmt.Errorf("Failed to write summary report: %s", err) + } + config.Conf.FormatJSON = before + config.Conf.Diff = beforeDiff + return nil +} + +func loadPrevious(current models.ScanResults) (previous models.ScanResults, err error) { + dirs, err := ListValidJSONDirs() + if err != nil { + return + } + + for _, result := range current { + for _, dir := range dirs[1:] { + var r models.ScanResult + path := filepath.Join(dir, result.ServerName+".json") + if r, err = loadOneServerScanResult(path); err != nil { + continue + } + if r.Family == result.Family && r.Release == result.Release { + previous = append(previous, r) + util.Log.Infof("Privious json found: %s", path) + break + } + } + } + return previous, nil +} + +func diff(curResults, preResults models.ScanResults) (diffed models.ScanResults, err error) { + for _, current := range curResults { + found := false + var previous models.ScanResult + for _, r := range preResults { + if current.ServerName == r.ServerName { + found = true + previous = r + break + } + } + + if found { + current.ScannedCves = getDiffCves(previous, current) + packages := models.Packages{} + for _, s := range current.ScannedCves { + for _, name := range s.PackageNames { + p := current.Packages[name] + packages[name] = p + } + } + current.Packages = packages + } + + diffed = append(diffed, current) + } + return diffed, err +} + +func getDiffCves(previous, current models.ScanResult) models.VulnInfos { + previousCveIDsSet := map[string]bool{} + for _, previousVulnInfo := range previous.ScannedCves { + previousCveIDsSet[previousVulnInfo.CveID] = true + } + + new := models.VulnInfos{} + updated := models.VulnInfos{} + for _, v := range current.ScannedCves { + if previousCveIDsSet[v.CveID] { + if isCveInfoUpdated(v.CveID, previous, current) { + updated[v.CveID] = v + } + } else { + new[v.CveID] = v + } + } + + for cveID, vuln := range new { + updated[cveID] = vuln + } + return updated +} + +func isCveInfoUpdated(cveID string, previous, current models.ScanResult) bool { + cTypes := []models.CveContentType{ + models.NVD, + models.JVN, + models.NewCveContentType(current.Family), + } + + prevLastModified := map[models.CveContentType]time.Time{} + for _, c := range previous.ScannedCves { + if cveID == c.CveID { + for _, cType := range cTypes { + content, _ := c.CveContents[cType] + prevLastModified[cType] = content.LastModified + } + break + } + } + + curLastModified := map[models.CveContentType]time.Time{} + for _, c := range current.ScannedCves { + if cveID == c.CveID { + for _, cType := range cTypes { + content, _ := c.CveContents[cType] + curLastModified[cType] = content.LastModified + } + break + } + } + for _, cType := range cTypes { + if equal := prevLastModified[cType].Equal(curLastModified[cType]); !equal { + return true + } + } + return false +} + +// jsonDirPattern is file name pattern of JSON directory +// 2016-11-16T10:43:28+09:00 +// 2016-11-16T10:43:28Z +var jsonDirPattern = regexp.MustCompile( + `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})$`) + +// ListValidJSONDirs returns valid json directory as array +// Returned array is sorted so that recent directories are at the head +func ListValidJSONDirs() (dirs []string, err error) { + var dirInfo []os.FileInfo + if dirInfo, err = ioutil.ReadDir(config.Conf.ResultsDir); err != nil { + err = fmt.Errorf("Failed to read %s: %s", + config.Conf.ResultsDir, err) + return + } + for _, d := range dirInfo { + if d.IsDir() && jsonDirPattern.MatchString(d.Name()) { + jsonDir := filepath.Join(config.Conf.ResultsDir, d.Name()) + dirs = append(dirs, jsonDir) + } + } + sort.Slice(dirs, func(i, j int) bool { + return dirs[j] < dirs[i] + }) + return +} + +// JSONDir returns +// If there is an arg, check if it is a valid format and return the corresponding path under results. +// If arg passed via PIPE (such as history subcommand), return that path. +// Otherwise, returns the path of the latest directory +func JSONDir(args []string) (string, error) { + var err error + dirs := []string{} + + if 0 < len(args) { + if dirs, err = ListValidJSONDirs(); err != nil { + return "", err + } + + path := filepath.Join(config.Conf.ResultsDir, args[0]) + for _, d := range dirs { + ss := strings.Split(d, string(os.PathSeparator)) + timedir := ss[len(ss)-1] + if timedir == args[0] { + return path, nil + } + } + + return "", fmt.Errorf("Invalid path: %s", path) + } + + // PIPE + if config.Conf.Pipe { + bytes, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return "", fmt.Errorf("Failed to read stdin: %s", err) + } + fields := strings.Fields(string(bytes)) + if 0 < len(fields) { + return filepath.Join(config.Conf.ResultsDir, fields[0]), nil + } + return "", fmt.Errorf("Stdin is invalid: %s", string(bytes)) + } + + // returns latest dir when no args or no PIPE + if dirs, err = ListValidJSONDirs(); err != nil { + return "", err + } + if len(dirs) == 0 { + return "", fmt.Errorf("No results under %s", + config.Conf.ResultsDir) + } + return dirs[0], nil +} + +// LoadScanResults read JSON data +func LoadScanResults(jsonDir string) (results models.ScanResults, err error) { + var files []os.FileInfo + if files, err = ioutil.ReadDir(jsonDir); err != nil { + return nil, fmt.Errorf("Failed to read %s: %s", jsonDir, err) + } + for _, f := range files { + if filepath.Ext(f.Name()) != ".json" || strings.HasSuffix(f.Name(), "_diff.json") { + continue + } + + var r models.ScanResult + path := filepath.Join(jsonDir, f.Name()) + if r, err = loadOneServerScanResult(path); err != nil { + return nil, err + } + + results = append(results, r) + } + if len(results) == 0 { + return nil, fmt.Errorf("There is no json file under %s", jsonDir) + } + return +} + +// loadOneServerScanResult read JSON data of one server +func loadOneServerScanResult(jsonFile string) (result models.ScanResult, err error) { + var data []byte + if data, err = ioutil.ReadFile(jsonFile); err != nil { + err = fmt.Errorf("Failed to read %s: %s", jsonFile, err) + return + } + if json.Unmarshal(data, &result) != nil { + err = fmt.Errorf("Failed to parse %s: %s", jsonFile, err) + } + return +} diff --git a/report/util_test.go b/report/util_test.go new file mode 100644 index 00000000..6cf3c1f9 --- /dev/null +++ b/report/util_test.go @@ -0,0 +1,327 @@ +package report + +import ( + "reflect" + "testing" + "time" + + "github.com/future-architect/vuls/models" + "github.com/k0kubun/pp" +) + +func TestIsCveInfoUpdated(t *testing.T) { + f := "2006-01-02" + old, _ := time.Parse(f, "2015-12-15") + new, _ := time.Parse(f, "2015-12-16") + + type In struct { + cveID string + cur models.ScanResult + prev models.ScanResult + } + var tests = []struct { + in In + expected bool + }{ + // NVD compare non-initialized times + { + in: In{ + cveID: "CVE-2017-0001", + cur: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0001", + LastModified: time.Time{}, + }, + ), + }, + }, + }, + prev: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2017-0001": { + CveID: "CVE-2017-0001", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0001", + LastModified: time.Time{}, + }, + ), + }, + }, + }, + }, + expected: false, + }, + // JVN not updated + { + in: In{ + cveID: "CVE-2017-0002", + cur: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0002", + LastModified: old, + }, + ), + }, + }, + }, + prev: models.ScanResult{ + ScannedCves: models.VulnInfos{ + "CVE-2017-0002": { + CveID: "CVE-2017-0002", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0002", + LastModified: old, + }, + ), + }, + }, + }, + }, + expected: false, + }, + // OVAL updated + { + in: In{ + cveID: "CVE-2017-0003", + cur: models.ScanResult{ + Family: "ubuntu", + ScannedCves: models.VulnInfos{ + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0002", + LastModified: new, + }, + ), + }, + }, + }, + prev: models.ScanResult{ + Family: "ubuntu", + ScannedCves: models.VulnInfos{ + "CVE-2017-0003": { + CveID: "CVE-2017-0003", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0002", + LastModified: old, + }, + ), + }, + }, + }, + }, + expected: true, + }, + // OVAL newly detected + { + in: In{ + cveID: "CVE-2017-0004", + cur: models.ScanResult{ + Family: "redhat", + ScannedCves: models.VulnInfos{ + "CVE-2017-0004": { + CveID: "CVE-2017-0004", + CveContents: models.NewCveContents( + models.CveContent{ + Type: models.NVD, + CveID: "CVE-2017-0002", + LastModified: old, + }, + ), + }, + }, + }, + prev: models.ScanResult{ + Family: "redhat", + ScannedCves: models.VulnInfos{}, + }, + }, + expected: true, + }, + } + for i, tt := range tests { + actual := isCveInfoUpdated(tt.in.cveID, tt.in.prev, tt.in.cur) + if actual != tt.expected { + t.Errorf("[%d] actual: %t, expected: %t", i, actual, tt.expected) + } + } +} + +func TestDiff(t *testing.T) { + atCurrent, _ := time.Parse("2006-01-02", "2014-12-31") + atPrevious, _ := time.Parse("2006-01-02", "2014-11-31") + var tests = []struct { + inCurrent models.ScanResults + inPrevious models.ScanResults + out models.ScanResult + }{ + { + inCurrent: models.ScanResults{ + { + ScannedAt: atCurrent, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + ScannedCves: models.VulnInfos{ + "CVE-2012-6702": { + CveID: "CVE-2012-6702", + PackageNames: []string{"libexpat1"}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + "CVE-2014-9761": { + CveID: "CVE-2014-9761", + PackageNames: []string{"libc-bin"}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + }, + Packages: models.Packages{}, + Errors: []string{}, + Optional: [][]interface{}{}, + }, + }, + inPrevious: models.ScanResults{ + { + ScannedAt: atPrevious, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + ScannedCves: models.VulnInfos{ + "CVE-2012-6702": { + CveID: "CVE-2012-6702", + PackageNames: []string{"libexpat1"}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + "CVE-2014-9761": { + CveID: "CVE-2014-9761", + PackageNames: []string{"libc-bin"}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + }, + Packages: models.Packages{}, + Errors: []string{}, + Optional: [][]interface{}{}, + }, + }, + out: models.ScanResult{ + ScannedAt: atCurrent, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + Packages: models.Packages{}, + ScannedCves: models.VulnInfos{}, + Errors: []string{}, + Optional: [][]interface{}{}, + }, + }, + { + inCurrent: models.ScanResults{ + { + ScannedAt: atCurrent, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + ScannedCves: models.VulnInfos{ + "CVE-2016-6662": { + CveID: "CVE-2016-6662", + PackageNames: []string{"mysql-libs"}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + }, + Packages: models.Packages{ + "mysql-libs": { + Name: "mysql-libs", + Version: "5.1.73", + Release: "7.el6", + NewVersion: "5.1.73", + NewRelease: "8.el6_8", + Repository: "", + Changelog: models.Changelog{ + Contents: "", + Method: "", + }, + }, + }, + }, + }, + inPrevious: models.ScanResults{ + { + ScannedAt: atPrevious, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + ScannedCves: models.VulnInfos{}, + }, + }, + out: models.ScanResult{ + ScannedAt: atCurrent, + ServerName: "u16", + Family: "ubuntu", + Release: "16.04", + ScannedCves: models.VulnInfos{ + "CVE-2016-6662": { + CveID: "CVE-2016-6662", + PackageNames: []string{"mysql-libs"}, + DistroAdvisories: []models.DistroAdvisory{}, + CpeNames: []string{}, + }, + }, + Packages: models.Packages{ + "mysql-libs": { + Name: "mysql-libs", + Version: "5.1.73", + Release: "7.el6", + NewVersion: "5.1.73", + NewRelease: "8.el6_8", + Repository: "", + Changelog: models.Changelog{ + Contents: "", + Method: "", + }, + }, + }, + }, + }, + } + + for i, tt := range tests { + diff, _ := diff(tt.inCurrent, tt.inPrevious) + for _, actual := range diff { + if !reflect.DeepEqual(actual.ScannedCves, tt.out.ScannedCves) { + h := pp.Sprint(actual.ScannedCves) + x := pp.Sprint(tt.out.ScannedCves) + t.Errorf("[%d] cves actual: \n %s \n expected: \n %s", i, h, x) + } + + for j := range tt.out.Packages { + if !reflect.DeepEqual(tt.out.Packages[j], actual.Packages[j]) { + h := pp.Sprint(tt.out.Packages[j]) + x := pp.Sprint(actual.Packages[j]) + t.Errorf("[%d] packages actual: \n %s \n expected: \n %s", i, x, h) + } + } + } + } +} diff --git a/util/util.go b/util/util.go index b7dcbc6e..31f27d9e 100644 --- a/util/util.go +++ b/util/util.go @@ -23,7 +23,6 @@ import ( "strings" "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/models" ) // GenWorkers generates goroutine @@ -139,18 +138,18 @@ func Truncate(str string, length int) string { // VendorLink returns a URL of the given OS family and CVEID //TODO -func VendorLink(family, cveID string) string { - cType := models.NewCveContentType(family) - switch cType { - case models.RedHat: - return "https://access.redhat.com/security/cve/" + cveID - case models.Debian: - return "https://security-tracker.debian.org/tracker/" + cveID - case models.Ubuntu: - return "http://people.ubuntu.com/~ubuntu-security/cve/" + cveID - // case models.FreeBSD: - // return "http://people.ubuntu.com/~ubuntu-security/cve/" + cveID - } +// func VendorLink(family, cveID string) string { +// cType := models.NewCveContentType(family) +// switch cType { +// case models.RedHat: +// return "https://access.redhat.com/security/cve/" + cveID +// case models.Debian: +// return "https://security-tracker.debian.org/tracker/" + cveID +// case models.Ubuntu: +// return "http://people.ubuntu.com/~ubuntu-security/cve/" + cveID +// // case models.FreeBSD: +// // return "http://people.ubuntu.com/~ubuntu-security/cve/" + cveID +// } - return "" -} +// return "" +// }