diff --git a/Makefile b/Makefile index 306dfb8d..f1e0d2ff 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ clean SRCS = $(shell git ls-files '*.go') -PKGS = ./. ./db ./config ./models ./report ./cveapi ./scan ./util ./commands +PKGS = ./. ./config ./models ./report ./cveapi ./scan ./util ./commands all: test diff --git a/README.ja.md b/README.ja.md index 289c321d..8675a611 100644 --- a/README.ja.md +++ b/README.ja.md @@ -601,8 +601,8 @@ scan: scan [-lang=en|ja] [-config=/path/to/config.toml] - [-dbpath=/path/to/vuls.sqlite3] - [--cve-dictionary-dbpath=/path/to/cve.sqlite3] + [-results-dir=/path/to/results] + [-cve-dictionary-dbpath=/path/to/cve.sqlite3] [-cve-dictionary-url=http://127.0.0.1:1323] [-cvss-over=7] [-ignore-unscored-cves] @@ -649,8 +649,8 @@ scan: http://CVE.Dictionary (default "http://127.0.0.1:1323") -cvss-over float -cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all)) - -dbpath string - /path/to/sqlite3 (default "$PWD/vuls.sqlite3") + -results-dir string + /path/to/results (default "$PWD/results") -debug debug mode -debug-sql @@ -892,10 +892,10 @@ Vulsは、DockerホストにSSHで接続し、`docker exec`でDockerコンテナ ``` $ vuls tui -h tui: - tui [-dbpath=/path/to/vuls.sqlite3] + tui [-results-dir=/path/to/results] - -dbpath string - /path/to/sqlite3 (default "$PWD/vuls.sqlite3") + -results-dir string + /path/to/results (default "$PWD/results") -debug-sql debug SQL diff --git a/README.md b/README.md index 4386aad6..41beb21a 100644 --- a/README.md +++ b/README.md @@ -599,8 +599,8 @@ scan: scan [-lang=en|ja] [-config=/path/to/config.toml] - [-dbpath=/path/to/vuls.sqlite3] - [--cve-dictionary-dbpath=/path/to/cve.sqlite3] + [-results-dir=/path/to/results] + [-cve-dictionary-dbpath=/path/to/cve.sqlite3] [-cve-dictionary-url=http://127.0.0.1:1323] [-cvss-over=7] [-ignore-unscored-cves] @@ -646,8 +646,8 @@ scan: http://CVE.Dictionary (default "http://127.0.0.1:1323") -cvss-over float -cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all)) - -dbpath string - /path/to/sqlite3 (default "$PWD/vuls.sqlite3") + -results-dir string + /path/to/results (default "$PWD/results") -debug debug mode -debug-sql @@ -882,10 +882,10 @@ For more details, see [Architecture section](https://github.com/future-architect ``` $ vuls tui -h tui: - tui [-dbpath=/path/to/vuls.sqlite3] + tui [-results-dir=/path/to/results] - -dbpath string - /path/to/sqlite3 (default "$PWD/vuls.sqlite3") + -results-dir string + /path/to/results (default "$PWD/results") -debug-sql debug SQL diff --git a/commands/history.go b/commands/history.go index 6e34dc6d..67508bed 100644 --- a/commands/history.go +++ b/commands/history.go @@ -20,25 +20,22 @@ package commands import ( "flag" "fmt" + "io/ioutil" "os" "path/filepath" "strings" - "golang.org/x/net/context" - - "github.com/Sirupsen/logrus" c "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/db" - "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/report" "github.com/google/subcommands" + "golang.org/x/net/context" ) // HistoryCmd is Subcommand of list scanned results type HistoryCmd struct { - debug bool - debugSQL bool - - dbpath string + debug bool + debugSQL bool + jsonBaseDir string } // Name return subcommand name @@ -53,7 +50,7 @@ func (*HistoryCmd) Synopsis() string { func (*HistoryCmd) Usage() string { return `history: history - [-dbpath=/path/to/vuls.sqlite3] + [-results-dir=/path/to/results] ` } @@ -62,47 +59,45 @@ func (p *HistoryCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.debugSQL, "debug-sql", false, "SQL debug mode") wd, _ := os.Getwd() - defaultDBPath := filepath.Join(wd, "vuls.sqlite3") - f.StringVar(&p.dbpath, "dbpath", defaultDBPath, "/path/to/sqlite3") + defaultJSONBaseDir := filepath.Join(wd, "results") + f.StringVar(&p.jsonBaseDir, "results-dir", defaultJSONBaseDir, "/path/to/results") } // Execute execute func (p *HistoryCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { c.Conf.DebugSQL = p.debugSQL - c.Conf.DBPath = p.dbpath + c.Conf.JSONBaseDir = p.jsonBaseDir - // _, err := scanHistories() - histories, err := scanHistories() - if err != nil { - logrus.Error("Failed to select scan histories: ", err) + var err error + var jsonDirs report.JSONDirs + if jsonDirs, err = report.GetValidJSONDirs(); err != nil { return subcommands.ExitFailure } - const timeLayout = "2006-01-02 15:04" - for _, history := range histories { - names := []string{} - for _, result := range history.ScanResults { - if 0 < len(result.Container.ContainerID) { - names = append(names, result.Container.Name) - } else { - names = append(names, result.ServerName) - } + for _, d := range jsonDirs { + var files []os.FileInfo + if files, err = ioutil.ReadDir(d); err != nil { + return subcommands.ExitFailure } - fmt.Printf("%-3d %s scanned %d servers: %s\n", - history.ID, - history.ScannedAt.Format(timeLayout), - len(history.ScanResults), - strings.Join(names, ", "), + var hosts []string + for _, f := range files { + // TODO this "if block" will be deleted in a future release + if f.Name() == "all.json" { + continue + } + if filepath.Ext(f.Name()) != ".json" { + continue + } + fileBase := strings.TrimSuffix(f.Name(), filepath.Ext(f.Name())) + hosts = append(hosts, fileBase) + } + splitPath := strings.Split(d, string(os.PathSeparator)) + timeStr := splitPath[len(splitPath)-1] + fmt.Printf("%s scanned %d servers: %s\n", + timeStr, + len(hosts), + strings.Join(hosts, ", "), ) } return subcommands.ExitSuccess } - -func scanHistories() (histories []models.ScanHistory, err error) { - if err := db.OpenDB(); err != nil { - return histories, fmt.Errorf( - "Failed to open DB. datafile: %s, err: %s", c.Conf.DBPath, err) - } - histories, err = db.SelectScanHistories() - return -} diff --git a/commands/scan.go b/commands/scan.go index 8d87d050..603cc706 100644 --- a/commands/scan.go +++ b/commands/scan.go @@ -24,11 +24,11 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/Sirupsen/logrus" c "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/cveapi" - "github.com/future-architect/vuls/db" "github.com/future-architect/vuls/report" "github.com/future-architect/vuls/scan" "github.com/future-architect/vuls/util" @@ -44,7 +44,7 @@ type ScanCmd struct { configPath string - dbpath string + jsonBaseDir string cvedbpath string cveDictionaryURL string @@ -86,7 +86,7 @@ func (*ScanCmd) Usage() string { scan [-lang=en|ja] [-config=/path/to/config.toml] - [-dbpath=/path/to/vuls.sqlite3] + [-results-dir=/path/to/results] [-cve-dictionary-dbpath=/path/to/cve.sqlite3] [-cve-dictionary-url=http://127.0.0.1:1323] [-cvss-over=7] @@ -124,8 +124,8 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { defaultConfPath := filepath.Join(wd, "config.toml") f.StringVar(&p.configPath, "config", defaultConfPath, "/path/to/toml") - defaultDBPath := filepath.Join(wd, "vuls.sqlite3") - f.StringVar(&p.dbpath, "dbpath", defaultDBPath, "/path/to/sqlite3") + defaultJSONBaseDir := filepath.Join(wd, "results") + f.StringVar(&p.jsonBaseDir, "results-dir", defaultJSONBaseDir, "/path/to/results") f.StringVar( &p.cvedbpath, @@ -284,6 +284,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) // logger Log := util.NewCustomLogger(c.ServerInfo{}) + scannedAt := time.Now() // report reports := []report.ResultWriter{ @@ -297,10 +298,10 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) reports = append(reports, report.MailWriter{}) } if p.reportJSON { - reports = append(reports, report.JSONWriter{}) + reports = append(reports, report.JSONWriter{ScannedAt: scannedAt}) } if p.reportText { - reports = append(reports, report.TextFileWriter{}) + reports = append(reports, report.TextFileWriter{ScannedAt: scannedAt}) } if p.reportS3 { c.Conf.AwsRegion = p.awsRegion @@ -315,17 +316,17 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } if p.reportAzureBlob { c.Conf.AzureAccount = p.azureAccount - if c.Conf.AzureAccount == "" { + if len(c.Conf.AzureAccount) == 0 { c.Conf.AzureAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") } c.Conf.AzureKey = p.azureKey - if c.Conf.AzureKey == "" { + if len(c.Conf.AzureKey) == 0 { c.Conf.AzureKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY") } c.Conf.AzureContainer = p.azureContainer - if c.Conf.AzureContainer == "" { + if len(c.Conf.AzureContainer) == 0 { Log.Error("Azure storage container name is requied with --azure-container option") return subcommands.ExitUsageError } @@ -337,7 +338,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) reports = append(reports, report.AzureBlobWriter{}) } - c.Conf.DBPath = p.dbpath + c.Conf.JSONBaseDir = p.jsonBaseDir c.Conf.CveDBPath = p.cvedbpath c.Conf.CveDictionaryURL = p.cveDictionaryURL c.Conf.CvssScoreOver = p.cvssScoreOver @@ -382,21 +383,6 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitFailure } - Log.Info("Insert to DB...") - if err := db.OpenDB(); err != nil { - Log.Errorf("Failed to open DB. datafile: %s, err: %s", c.Conf.DBPath, err) - return subcommands.ExitFailure - } - if err := db.MigrateDB(); err != nil { - Log.Errorf("Failed to migrate. err: %s", err) - return subcommands.ExitFailure - } - - if err := db.Insert(scanResults); err != nil { - Log.Fatalf("Failed to insert. dbpath: %s, err: %s", c.Conf.DBPath, err) - return subcommands.ExitFailure - } - Log.Info("Reporting...") filtered := scanResults.FilterByCvssOver() for _, w := range reports { diff --git a/commands/tui.go b/commands/tui.go index 8bb3a4fa..4112b598 100644 --- a/commands/tui.go +++ b/commands/tui.go @@ -19,11 +19,9 @@ package commands import ( "flag" - "fmt" "io/ioutil" "os" "path/filepath" - "strconv" "strings" log "github.com/Sirupsen/logrus" @@ -35,9 +33,9 @@ import ( // TuiCmd is Subcommand of host discovery mode type TuiCmd struct { - lang string - debugSQL bool - dbpath string + lang string + debugSQL bool + jsonBaseDir string } // Name return subcommand name @@ -49,7 +47,7 @@ func (*TuiCmd) Synopsis() string { return "Run Tui view to anayze vulnerabilites // Usage return usage func (*TuiCmd) Usage() string { return `tui: - tui [-dbpath=/path/to/vuls.sqlite3] + tui [-results-dir=/path/to/results] ` } @@ -61,24 +59,34 @@ func (p *TuiCmd) SetFlags(f *flag.FlagSet) { wd, _ := os.Getwd() - defaultDBPath := filepath.Join(wd, "vuls.sqlite3") - f.StringVar(&p.dbpath, "dbpath", defaultDBPath, - fmt.Sprintf("/path/to/sqlite3 (default: %s)", defaultDBPath)) + defaultJSONBaseDir := filepath.Join(wd, "results") + f.StringVar(&p.jsonBaseDir, "results-dir", defaultJSONBaseDir, "/path/to/results") } // Execute execute func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { c.Conf.Lang = "en" c.Conf.DebugSQL = p.debugSQL - c.Conf.DBPath = p.dbpath + c.Conf.JSONBaseDir = p.jsonBaseDir - historyID := "" + var jsonDirName string + var err error if 0 < len(f.Args()) { - if _, err := strconv.Atoi(f.Args()[0]); err != nil { - log.Errorf("First Argument have to be scan_histores record ID: %s", err) + var jsonDirs report.JSONDirs + if jsonDirs, err = report.GetValidJSONDirs(); err != nil { + return subcommands.ExitFailure + } + for _, d := range jsonDirs { + splitPath := strings.Split(d, string(os.PathSeparator)) + if splitPath[len(splitPath)-1] == f.Args()[0] { + jsonDirName = f.Args()[0] + break + } + } + if len(jsonDirName) == 0 { + log.Errorf("First Argument have to be JSON directory name : %s", err) return subcommands.ExitFailure } - historyID = f.Args()[0] } else { stat, _ := os.Stdin.Stat() if (stat.Mode() & os.ModeCharDevice) == 0 { @@ -89,9 +97,9 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s } fields := strings.Fields(string(bytes)) if 0 < len(fields) { - historyID = fields[0] + jsonDirName = fields[0] } } } - return report.RunTui(historyID) + return report.RunTui(jsonDirName) } diff --git a/config/config.go b/config/config.go index 13c6e677..35cf89da 100644 --- a/config/config.go +++ b/config/config.go @@ -46,9 +46,9 @@ type Config struct { SSHExternal bool - HTTPProxy string `valid:"url"` - DBPath string - CveDBPath string + HTTPProxy string `valid:"url"` + JSONBaseDir string + CveDBPath string AwsProfile string AwsRegion string @@ -66,10 +66,10 @@ type Config struct { func (c Config) Validate() bool { errs := []error{} - if len(c.DBPath) != 0 { - if ok, _ := valid.IsFilePath(c.DBPath); !ok { + if len(c.JSONBaseDir) != 0 { + if ok, _ := valid.IsFilePath(c.JSONBaseDir); !ok { errs = append(errs, fmt.Errorf( - "SQLite3 DB path must be a *Absolute* file path. dbpath: %s", c.DBPath)) + "JSON base directory must be a *Absolute* file path. jsonBaseDir: %s", c.JSONBaseDir)) } } diff --git a/config/tomlloader.go b/config/tomlloader.go index 06c88478..0ca05f8d 100644 --- a/config/tomlloader.go +++ b/config/tomlloader.go @@ -68,7 +68,7 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { } s.Host = v.Host - if s.Host == "" { + if len(s.Host) == 0 { return fmt.Errorf("%s is invalid. host is empty", name) } @@ -82,7 +82,7 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { } s.KeyPath = v.KeyPath - if s.KeyPath == "" { + if len(s.KeyPath) == 0 { s.KeyPath = d.KeyPath } if s.KeyPath != "" { @@ -94,7 +94,7 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) (err error) { // s.KeyPassword = keyPass s.KeyPassword = v.KeyPassword - if s.KeyPassword == "" { + if len(s.KeyPassword) == 0 { s.KeyPassword = d.KeyPassword } diff --git a/cveapi/cve_client.go b/cveapi/cve_client.go index bb22c9bd..22a0db29 100644 --- a/cveapi/cve_client.go +++ b/cveapi/cve_client.go @@ -59,7 +59,7 @@ func (api cvedictClient) CheckHealth() (ok bool, err error) { var resp *http.Response resp, _, errs = gorequest.New().SetDebug(config.Conf.Debug).Get(url).End() // resp, _, errs = gorequest.New().Proxy(api.httpProxy).Get(url).End() - if len(errs) > 0 || resp == nil || resp.StatusCode != 200 { + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { return false, fmt.Errorf("Failed to request to CVE server. url: %s, errs: %v", url, errs) } return true, nil @@ -198,7 +198,7 @@ func (api cvedictClient) httpGet(key, url string, resChan chan<- response, errCh // } // pp.Println(req) // resp, body, errs = req.End() -// if len(errs) > 0 || resp.StatusCode != 200 { +// if 0 < len(errs) || resp.StatusCode != 200 { // errChan <- fmt.Errorf("HTTP error. errs: %v, url: %s", errs, url) // } // return nil @@ -252,7 +252,7 @@ func (api cvedictClient) httpPost(key, url string, query map[string]string) ([]c req = req.Send(fmt.Sprintf("%s=%s", key, query[key])).Type("json") } resp, body, errs = req.End() - if len(errs) > 0 || resp == nil || resp.StatusCode != 200 { + if 0 < len(errs) || resp == nil || resp.StatusCode != 200 { return fmt.Errorf("HTTP POST error: %v, url: %s, resp: %v", errs, url, resp) } return nil diff --git a/db/db.go b/db/db.go deleted file mode 100644 index 27ca983f..00000000 --- a/db/db.go +++ /dev/null @@ -1,324 +0,0 @@ -/* Vuls - Vulnerability Scanner -Copyright (C) 2016 Future Architect, Inc. Japan. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -*/ - -package db - -import ( - "fmt" - "sort" - "strconv" - "time" - - "github.com/future-architect/vuls/config" - m "github.com/future-architect/vuls/models" - "github.com/jinzhu/gorm" - cvedb "github.com/kotakanbe/go-cve-dictionary/db" - cve "github.com/kotakanbe/go-cve-dictionary/models" -) - -var db *gorm.DB - -// OpenDB opens Database -func OpenDB() (err error) { - db, err = gorm.Open("sqlite3", config.Conf.DBPath) - if err != nil { - err = fmt.Errorf("Failed to open DB. datafile: %s, err: %s", config.Conf.DBPath, err) - return - - } - db.LogMode(config.Conf.DebugSQL) - return -} - -// MigrateDB migrates Database -func MigrateDB() error { - if err := db.AutoMigrate( - &m.ScanHistory{}, - &m.ScanResult{}, - // &m.NWLink{}, - &m.Container{}, - &m.CveInfo{}, - &m.CpeName{}, - &m.PackageInfo{}, - &m.DistroAdvisory{}, - &cve.CveDetail{}, - &cve.Jvn{}, - &cve.Nvd{}, - &cve.Reference{}, - &cve.Cpe{}, - ).Error; err != nil { - return fmt.Errorf("Failed to migrate. err: %s", err) - } - - errMsg := "Failed to create index. err: %s" - // if err := db.Model(&m.NWLink{}). - // AddIndex("idx_n_w_links_scan_result_id", "scan_result_id").Error; err != nil { - // return fmt.Errorf(errMsg, err) - // } - if err := db.Model(&m.Container{}). - AddIndex("idx_containers_scan_result_id", "scan_result_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&m.CveInfo{}). - AddIndex("idx_cve_infos_scan_result_id", "scan_result_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&m.CpeName{}). - AddIndex("idx_cpe_names_cve_info_id", "cve_info_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&m.PackageInfo{}). - AddIndex("idx_package_infos_cve_info_id", "cve_info_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&m.DistroAdvisory{}). - //TODO check table name - AddIndex("idx_distro_advisories_cve_info_id", "cve_info_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&cve.CveDetail{}). - AddIndex("idx_cve_details_cve_info_id", "cve_info_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&cve.CveDetail{}). - AddIndex("idx_cve_details_cveid", "cve_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&cve.Nvd{}). - AddIndex("idx_nvds_cve_detail_id", "cve_detail_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&cve.Jvn{}). - AddIndex("idx_jvns_cve_detail_id", "cve_detail_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&cve.Cpe{}). - AddIndex("idx_cpes_jvn_id", "jvn_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&cve.Reference{}). - AddIndex("idx_references_jvn_id", "jvn_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&cve.Cpe{}). - AddIndex("idx_cpes_nvd_id", "nvd_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - if err := db.Model(&cve.Reference{}). - AddIndex("idx_references_nvd_id", "nvd_id").Error; err != nil { - return fmt.Errorf(errMsg, err) - } - - return nil -} - -// Insert inserts scan results into DB -func Insert(results []m.ScanResult) error { - for _, r := range results { - r.KnownCves = resetGormIDs(r.KnownCves) - r.UnknownCves = resetGormIDs(r.UnknownCves) - } - - history := m.ScanHistory{ - ScanResults: results, - ScannedAt: time.Now(), - } - - db = db.Set("gorm:save_associations", false) - if err := db.Create(&history).Error; err != nil { - return err - } - for _, scanResult := range history.ScanResults { - scanResult.ScanHistoryID = history.ID - if err := db.Create(&scanResult).Error; err != nil { - return err - } - scanResult.Container.ScanResultID = scanResult.ID - if err := db.Create(&scanResult.Container).Error; err != nil { - return err - } - if err := insertCveInfos(scanResult.ID, scanResult.KnownCves); err != nil { - return err - } - if err := insertCveInfos(scanResult.ID, scanResult.UnknownCves); err != nil { - return err - } - } - return nil -} - -func insertCveInfos(scanResultID uint, infos []m.CveInfo) error { - for _, cveInfo := range infos { - cveInfo.ScanResultID = scanResultID - if err := db.Create(&cveInfo).Error; err != nil { - return err - } - - for _, pack := range cveInfo.Packages { - pack.CveInfoID = cveInfo.ID - if err := db.Create(&pack).Error; err != nil { - return err - } - } - - for _, distroAdvisory := range cveInfo.DistroAdvisories { - distroAdvisory.CveInfoID = cveInfo.ID - if err := db.Create(&distroAdvisory).Error; err != nil { - return err - } - } - - for _, cpeName := range cveInfo.CpeNames { - cpeName.CveInfoID = cveInfo.ID - if err := db.Create(&cpeName).Error; err != nil { - return err - } - } - - db = db.Set("gorm:save_associations", true) - cveDetail := cveInfo.CveDetail - cveDetail.CveInfoID = cveInfo.ID - if err := db.Create(&cveDetail).Error; err != nil { - return err - } - db = db.Set("gorm:save_associations", false) - } - return nil -} - -func resetGormIDs(infos []m.CveInfo) []m.CveInfo { - for i := range infos { - infos[i].CveDetail.ID = 0 - // NVD - infos[i].CveDetail.Nvd.ID = 0 - for j := range infos[i].CveDetail.Nvd.Cpes { - infos[i].CveDetail.Nvd.Cpes[j].ID = 0 - } - for j := range infos[i].CveDetail.Nvd.References { - infos[i].CveDetail.Nvd.References[j].ID = 0 - } - - // JVN - infos[i].CveDetail.Jvn.ID = 0 - for j := range infos[i].CveDetail.Jvn.Cpes { - infos[i].CveDetail.Jvn.Cpes[j].ID = 0 - } - for j := range infos[i].CveDetail.Jvn.References { - infos[i].CveDetail.Jvn.References[j].ID = 0 - } - - //Packages - for j := range infos[i].Packages { - infos[i].Packages[j].ID = 0 - infos[i].Packages[j].CveInfoID = 0 - } - } - return infos -} - -// SelectScanHistory select scan history from DB -func SelectScanHistory(historyID string) (m.ScanHistory, error) { - var err error - - scanHistory := m.ScanHistory{} - if historyID == "" { - // select latest - db.Order("scanned_at desc").First(&scanHistory) - } else { - var id int - if id, err = strconv.Atoi(historyID); err != nil { - return m.ScanHistory{}, - fmt.Errorf("historyID have to be numeric number: %s", err) - } - db.First(&scanHistory, id) - } - - if scanHistory.ID == 0 { - return m.ScanHistory{}, fmt.Errorf("No scanHistory records") - } - - // results := []m.ScanResult{} - results := m.ScanResults{} - db.Model(&scanHistory).Related(&results, "ScanResults") - scanHistory.ScanResults = results - - for i, r := range results { - // nw := []m.NWLink{} - // db.Model(&r).Related(&nw, "NWLinks") - // scanHistory.ScanResults[i].NWLinks = nw - - di := m.Container{} - db.Model(&r).Related(&di, "Container") - scanHistory.ScanResults[i].Container = di - - knownCves := selectCveInfos(&r, "KnownCves") - sort.Sort(m.CveInfos(knownCves)) - scanHistory.ScanResults[i].KnownCves = knownCves - } - - sort.Sort(scanHistory.ScanResults) - return scanHistory, nil -} - -func selectCveInfos(result *m.ScanResult, fieldName string) []m.CveInfo { - cveInfos := []m.CveInfo{} - db.Model(&result).Related(&cveInfos, fieldName) - - for i, cveInfo := range cveInfos { - cveDetail := cve.CveDetail{} - db.Model(&cveInfo).Related(&cveDetail, "CveDetail") - id := cveDetail.CveID - filledCveDetail := cvedb.Get(id, db) - cveInfos[i].CveDetail = filledCveDetail - - packs := []m.PackageInfo{} - db.Model(&cveInfo).Related(&packs, "Packages") - cveInfos[i].Packages = packs - - advisories := []m.DistroAdvisory{} - db.Model(&cveInfo).Related(&advisories, "DistroAdvisories") - cveInfos[i].DistroAdvisories = advisories - - names := []m.CpeName{} - db.Model(&cveInfo).Related(&names, "CpeNames") - cveInfos[i].CpeNames = names - } - return cveInfos -} - -// SelectScanHistories select latest scan history from DB -func SelectScanHistories() ([]m.ScanHistory, error) { - scanHistories := []m.ScanHistory{} - db.Order("scanned_at desc").Find(&scanHistories) - - if len(scanHistories) == 0 { - return []m.ScanHistory{}, fmt.Errorf("No scanHistory records") - } - - for i, history := range scanHistories { - results := m.ScanResults{} - db.Model(&history).Related(&results, "ScanResults") - scanHistories[i].ScanResults = results - - for j, r := range results { - di := m.Container{} - db.Model(&r).Related(&di, "Container") - scanHistories[i].ScanResults[j].Container = di - } - } - return scanHistories, nil -} diff --git a/glide.lock b/glide.lock index 282caddd..77208989 100644 --- a/glide.lock +++ b/glide.lock @@ -63,7 +63,6 @@ imports: version: 1a336b8ac785badfe89a175ee926d39574901232 subpackages: - config - - db - models - log - jvn diff --git a/glide.yaml b/glide.yaml index 9d328dd9..015240a5 100644 --- a/glide.yaml +++ b/glide.yaml @@ -22,7 +22,6 @@ import: - package: github.com/kotakanbe/go-cve-dictionary subpackages: - config - - db - models - package: github.com/kotakanbe/go-pingscanner - package: github.com/kotakanbe/logrus-prefixed-formatter diff --git a/models/models.go b/models/models.go index 1fbe3726..7c862cfe 100644 --- a/models/models.go +++ b/models/models.go @@ -74,6 +74,7 @@ func (s ScanResults) FilterByCvssOver() (filtered ScanResults) { type ScanResult struct { gorm.Model `json:"-"` ScanHistoryID uint `json:"-"` + ScannedAt time.Time ServerName string // TOML Section key // Hostname string @@ -95,7 +96,7 @@ type ScanResult struct { // ServerInfo returns server name one line func (r ScanResult) ServerInfo() string { hostinfo := "" - if r.Container.ContainerID == "" { + if len(r.Container.ContainerID) == 0 { hostinfo = fmt.Sprintf( "%s (%s%s)", r.ServerName, @@ -118,7 +119,7 @@ func (r ScanResult) ServerInfo() string { // ServerInfoTui returns server infromation for TUI sidebar func (r ScanResult) ServerInfoTui() string { hostinfo := "" - if r.Container.ContainerID == "" { + if len(r.Container.ContainerID) == 0 { hostinfo = fmt.Sprintf( "%s (%s%s)", r.ServerName, @@ -190,7 +191,7 @@ func (c CveInfos) Less(i, j int) bool { if c[i].CveDetail.CvssScore(lang) == c[j].CveDetail.CvssScore(lang) { return c[i].CveDetail.CveID < c[j].CveDetail.CveID } - return c[i].CveDetail.CvssScore(lang) > c[j].CveDetail.CvssScore(lang) + return c[j].CveDetail.CvssScore(lang) < c[i].CveDetail.CvssScore(lang) } // CveInfo has Cve Information. diff --git a/report/azureblob.go b/report/azureblob.go index e44187f0..4991b78d 100644 --- a/report/azureblob.go +++ b/report/azureblob.go @@ -115,7 +115,7 @@ func (w AzureBlobWriter) upload(res models.ScanResult) (err error) { } timestr := time.Now().Format("20060102_1504") name := "" - if res.Container.ContainerID == "" { + if len(res.Container.ContainerID) == 0 { name = fmt.Sprintf("%s/%s.json", timestr, res.ServerName) } else { name = fmt.Sprintf("%s/%s_%s.json", timestr, res.ServerName, res.Container.Name) diff --git a/report/json.go b/report/json.go index 92d86d95..415df74f 100644 --- a/report/json.go +++ b/report/json.go @@ -21,17 +21,44 @@ import ( "encoding/json" "fmt" "io/ioutil" + "os" "path/filepath" + "regexp" + "sort" + "strings" + "time" + c "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" ) +// JSONDirs array of json files path. +type JSONDirs []string + +func (d JSONDirs) Len() int { + return len(d) +} +func (d JSONDirs) Swap(i, j int) { + d[i], d[j] = d[j], d[i] +} +func (d JSONDirs) Less(i, j int) bool { + return d[j] < d[i] +} + // JSONWriter writes results to file. -type JSONWriter struct{} +type JSONWriter struct { + ScannedAt time.Time +} func (w JSONWriter) Write(scanResults []models.ScanResult) (err error) { + var path string + if path, err = ensureResultDir(w.ScannedAt); err != nil { + return fmt.Errorf("Failed to make direcotory/symlink : %s", err) + } - path, err := ensureResultDir() + for _, scanResult := range scanResults { + scanResult.ScannedAt = w.ScannedAt + } var jsonBytes []byte if jsonBytes, err = json.Marshal(scanResults); err != nil { @@ -44,7 +71,7 @@ func (w JSONWriter) Write(scanResults []models.ScanResult) (err error) { for _, r := range scanResults { jsonPath := "" - if r.Container.ContainerID == "" { + if len(r.Container.ContainerID) == 0 { jsonPath = filepath.Join(path, fmt.Sprintf("%s.json", r.ServerName)) } else { jsonPath = filepath.Join(path, @@ -60,3 +87,77 @@ func (w JSONWriter) Write(scanResults []models.ScanResult) (err error) { } return nil } + +// JSONDirPattern is file name pattern of JSON directory +var JSONDirPattern = regexp.MustCompile(`^\d{8}_\d{4}$`) + +// GetValidJSONDirs return valid json directory as array +func GetValidJSONDirs() (jsonDirs JSONDirs, err error) { + var dirInfo []os.FileInfo + if dirInfo, err = ioutil.ReadDir(c.Conf.JSONBaseDir); err != nil { + err = fmt.Errorf("Failed to read %s: %s", c.Conf.JSONBaseDir, err) + return + } + for _, d := range dirInfo { + if d.IsDir() && JSONDirPattern.MatchString(d.Name()) { + jsonDir := filepath.Join(c.Conf.JSONBaseDir, d.Name()) + jsonDirs = append(jsonDirs, jsonDir) + } + } + sort.Sort(jsonDirs) + return +} + +// LoadOneScanHistory read JSON data +func LoadOneScanHistory(jsonDir string) (scanHistory models.ScanHistory, err error) { + var scanResults []models.ScanResult + var files []os.FileInfo + if files, err = ioutil.ReadDir(jsonDir); err != nil { + err = fmt.Errorf("Failed to read %s: %s", jsonDir, err) + return + } + for _, file := range files { + // TODO this "if block" will be deleted in a future release + if file.Name() == "all.json" { + continue + } + if filepath.Ext(file.Name()) != ".json" { + continue + } + var scanResult models.ScanResult + var data []byte + jsonPath := filepath.Join(jsonDir, file.Name()) + if data, err = ioutil.ReadFile(jsonPath); err != nil { + err = fmt.Errorf("Failed to read %s: %s", jsonPath, err) + return + } + if json.Unmarshal(data, &scanResult) != nil { + err = fmt.Errorf("Failed to parse %s: %s", jsonPath, err) + return + } + scanResults = append(scanResults, scanResult) + } + if len(scanResults) == 0 { + err = fmt.Errorf("There is no json file under %s", jsonDir) + return + } + + var scannedAt time.Time + if scanResults[0].ScannedAt.IsZero() { + splitPath := strings.Split(jsonDir, string(os.PathSeparator)) + timeStr := splitPath[len(splitPath)-1] + timeformat := "20060102_1504" + if scannedAt, err = time.Parse(timeformat, timeStr); err != nil { + err = fmt.Errorf("Failed to parse %s: %s", timeStr, err) + return + } + } else { + scannedAt = scanResults[0].ScannedAt + } + + scanHistory = models.ScanHistory{ + ScanResults: scanResults, + ScannedAt: scannedAt, + } + return +} diff --git a/report/s3.go b/report/s3.go index b0f87711..95b7951e 100644 --- a/report/s3.go +++ b/report/s3.go @@ -90,7 +90,7 @@ func (w S3Writer) Write(scanResults []models.ScanResult) (err error) { for _, r := range scanResults { key := "" - if r.Container.ContainerID == "" { + if len(r.Container.ContainerID) == 0 { key = fmt.Sprintf("%s/%s.json", timestr, r.ServerName) } else { key = fmt.Sprintf("%s/%s_%s.json", timestr, r.ServerName, r.Container.Name) diff --git a/report/slack.go b/report/slack.go index ca583279..7b3d5c46 100644 --- a/report/slack.go +++ b/report/slack.go @@ -80,7 +80,7 @@ func (w SlackWriter) Write(scanResults []models.ScanResult) error { Send(string(jsonBody)).End() if resp.StatusCode != 200 { log.Errorf("Resonse body: %s", body) - if len(errs) > 0 { + if 0 < len(errs) { return errs[0] } } diff --git a/report/textfile.go b/report/textfile.go index 34685e8d..0d0cdead 100644 --- a/report/textfile.go +++ b/report/textfile.go @@ -22,21 +22,22 @@ import ( "io/ioutil" "path/filepath" "strings" + "time" "github.com/future-architect/vuls/models" ) // TextFileWriter writes results to file. -type TextFileWriter struct{} +type TextFileWriter struct { + ScannedAt time.Time +} func (w TextFileWriter) Write(scanResults []models.ScanResult) (err error) { - - path, err := ensureResultDir() - + path, err := ensureResultDir(w.ScannedAt) all := []string{} for _, r := range scanResults { textFilePath := "" - if r.Container.ContainerID == "" { + if len(r.Container.ContainerID) == 0 { textFilePath = filepath.Join(path, fmt.Sprintf("%s.txt", r.ServerName)) } else { textFilePath = filepath.Join(path, diff --git a/report/tui.go b/report/tui.go index 693ba901..c8b4b291 100644 --- a/report/tui.go +++ b/report/tui.go @@ -20,13 +20,13 @@ package report import ( "bytes" "fmt" + "path/filepath" "strings" "text/template" "time" log "github.com/Sirupsen/logrus" "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/db" "github.com/future-architect/vuls/models" "github.com/google/subcommands" "github.com/gosuri/uitable" @@ -40,9 +40,9 @@ var currentCveInfo int var currentDetailLimitY int // RunTui execute main logic -func RunTui(historyID string) subcommands.ExitStatus { +func RunTui(jsonDirName string) subcommands.ExitStatus { var err error - scanHistory, err = selectScanHistory(historyID) + scanHistory, err = selectScanHistory(jsonDirName) if err != nil { log.Fatal(err) return subcommands.ExitFailure @@ -70,12 +70,20 @@ func RunTui(historyID string) subcommands.ExitStatus { return subcommands.ExitSuccess } -func selectScanHistory(historyID string) (latest models.ScanHistory, err error) { - if err := db.OpenDB(); err != nil { - return latest, fmt.Errorf( - "Failed to open DB. datafile: %s, err: %s", config.Conf.DBPath, err) +func selectScanHistory(jsonDirName string) (latest models.ScanHistory, err error) { + var jsonDir string + if 0 < len(jsonDirName) { + jsonDir = filepath.Join(config.Conf.JSONBaseDir, jsonDirName) + } else { + var jsonDirs JSONDirs + if jsonDirs, err = GetValidJSONDirs(); err != nil { + return + } + jsonDir = jsonDirs[0] + } + if latest, err = LoadOneScanHistory(jsonDir); err != nil { + return } - latest, err = db.SelectScanHistory(historyID) return } @@ -332,7 +340,7 @@ func cursorUp(g *gocui.Gui, v *gocui.View) error { if v != nil { ox, oy := v.Origin() cx, cy := v.Cursor() - if err := v.SetCursor(cx, cy-1); err != nil && oy > 0 { + if err := v.SetCursor(cx, cy-1); err != nil && 0 < oy { if err := v.SetOrigin(ox, oy-1); err != nil { return err } diff --git a/report/util.go b/report/util.go index 4875c92d..72ba4d80 100644 --- a/report/util.go +++ b/report/util.go @@ -30,13 +30,13 @@ import ( "github.com/gosuri/uitable" ) -func ensureResultDir() (path string, err error) { +func ensureResultDir(scannedAt time.Time) (path string, err error) { if resultDirPath != "" { return resultDirPath, nil } const timeLayout = "20060102_1504" - timedir := time.Now().Format(timeLayout) + timedir := scannedAt.Format(timeLayout) wd, _ := os.Getwd() dir := filepath.Join(wd, "results", timedir) if err := os.MkdirAll(dir, 0755); err != nil { @@ -44,7 +44,7 @@ func ensureResultDir() (path string, err error) { } symlinkPath := filepath.Join(wd, "results", "current") - if _, err := os.Stat(symlinkPath); err == nil { + if _, err := os.Lstat(symlinkPath); err == nil { if err := os.Remove(symlinkPath); err != nil { return "", fmt.Errorf( "Failed to remove symlink. path: %s, err: %s", symlinkPath, err) diff --git a/scan/base.go b/scan/base.go index a8c4baac..da3f4dd9 100644 --- a/scan/base.go +++ b/scan/base.go @@ -21,6 +21,7 @@ import ( "fmt" "sort" "strings" + "time" "github.com/Sirupsen/logrus" "github.com/future-architect/vuls/config" @@ -239,6 +240,7 @@ func (l *base) convertToModel() (models.ScanResult, error) { return models.ScanResult{ ServerName: l.ServerInfo.ServerName, + ScannedAt: time.Now(), Family: l.Family, Release: l.Release, Container: container, diff --git a/scan/debian.go b/scan/debian.go index e352ea03..8bc75aaa 100644 --- a/scan/debian.go +++ b/scan/debian.go @@ -522,7 +522,7 @@ func (o *debian) parseChangelog(changelog string, o.log.Debugf("Found the stop line. line: %s", line) stopLineFound = true break - } else if matches := cveRe.FindAllString(line, -1); len(matches) > 0 { + } else if matches := cveRe.FindAllString(line, -1); 0 < len(matches) { for _, m := range matches { cveIDs = util.AppendIfMissing(cveIDs, m) } diff --git a/scan/redhat.go b/scan/redhat.go index 2faa764c..fb018468 100644 --- a/scan/redhat.go +++ b/scan/redhat.go @@ -478,7 +478,7 @@ func (o *redhat) parseAllChangelog(allChangelog string) (map[string]*string, err /* for CentOS5 (yum-util < 1.1.20) */ prev = false now = false - if i > 0 { + if 0 < i { prev, err = o.isRpmPackageNameLine(orglines[i-1]) if err != nil { return nil, err diff --git a/scan/sshutil.go b/scan/sshutil.go index 7c38ec57..55190fc8 100644 --- a/scan/sshutil.go +++ b/scan/sshutil.go @@ -319,7 +319,7 @@ func decolateCmd(c conf.ServerInfo, cmd string, sudo bool) string { } func getAgentAuth() (auth ssh.AuthMethod, ok bool) { - if sock := os.Getenv("SSH_AUTH_SOCK"); len(sock) > 0 { + if sock := os.Getenv("SSH_AUTH_SOCK"); 0 < len(sock) { if agconn, err := net.Dial("unix", sock); err == nil { ag := agent.NewClient(agconn) auth = ssh.PublicKeysCallback(ag.Signers) diff --git a/util/util.go b/util/util.go index ecda23e3..74781e99 100644 --- a/util/util.go +++ b/util/util.go @@ -111,7 +111,7 @@ func ProxyEnv() string { // PrependProxyEnv prepends proxy enviroment variable func PrependProxyEnv(cmd string) string { - if config.Conf.HTTPProxy == "" { + if len(config.Conf.HTTPProxy) == 0 { return cmd } return fmt.Sprintf("%s %s", ProxyEnv(), cmd)