From 03579126fd1797f7c6647d12f51d54d1a7b478cd Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Thu, 25 Feb 2021 05:54:17 +0900 Subject: [PATCH] refactor(config): localize config used like a global variable (#1179) * refactor(report): LocalFileWriter * refactor -format-json * refacotr: -format-one-email * refactor: -format-csv * refactor: -gzip * refactor: -format-full-text * refactor: -format-one-line-text * refactor: -format-list * refacotr: remove -to-* from config * refactor: IgnoreGitHubDismissed * refactor: GitHub * refactor: IgnoreUnsocred * refactor: diff * refacotr: lang * refacotr: cacheDBPath * refactor: Remove config references * refactor: ScanResults * refacotr: constant pkg * chore: comment * refactor: scanner * refactor: scanner * refactor: serverapi.go * refactor: serverapi * refactor: change pkg structure * refactor: serverapi.go * chore: remove emtpy file * fix(scan): remove -ssh-native-insecure option * fix(scan): remove the deprecated option `keypassword` --- config/awsconf.go | 30 + config/azureconf.go | 24 + config/chatworkconf.go | 3 +- config/config.go | 166 ++-- config/config_test.go | 4 +- config/exploitconf.go | 2 +- config/gocvedictconf.go | 2 +- config/gostconf.go | 2 +- config/govaldictconf.go | 2 +- config/httpconf.go | 11 +- config/ips.go | 5 +- config/metasploitconf.go | 2 +- config/os.go | 73 +- config/os_test.go | 2 + config/slackconf.go | 3 +- config/smtpconf.go | 8 +- config/syslogconf.go | 3 +- config/telegramconf.go | 7 +- config/tomlloader.go | 15 +- constant/constant.go | 58 ++ {report => detector}/cve_client.go | 2 +- {report => detector}/db_client.go | 12 +- report/report.go => detector/detector.go | 91 +- {github => detector}/github.go | 7 +- .../libManager.go => detector/library.go | 9 +- detector/util.go | 270 ++++++ {wordpress => detector}/wordpress.go | 5 +- {wordpress => detector}/wordpress_test.go | 2 +- go.mod | 1 - gost/debian.go | 3 +- gost/gost.go | 9 +- models/scanresults.go | 311 +++--- models/scanresults_test.go | 230 +---- models/vulninfos.go | 22 +- oval/alpine.go | 3 +- oval/debian.go | 11 +- oval/redhat.go | 11 +- oval/suse.go | 3 +- oval/util.go | 33 +- oval/util_test.go | 12 +- {report => reporter}/azureblob.go | 30 +- {report => reporter}/chatwork.go | 4 +- {report => reporter}/email.go | 32 +- {report => reporter}/http.go | 2 +- {report => reporter}/localfile.go | 47 +- {report => reporter}/s3.go | 30 +- {report => reporter}/slack.go | 47 +- {report => reporter}/slack_test.go | 4 +- {report => reporter}/stdout.go | 18 +- {report => reporter}/syslog.go | 2 +- {report => reporter}/syslog_test.go | 2 +- {report => reporter}/telegram.go | 6 +- {report => reporter}/util.go | 127 +-- {report => reporter}/util_test.go | 103 +- {report => reporter}/writer.go | 2 +- scan/utils.go | 36 - {scan => scanner}/alpine.go | 5 +- {scan => scanner}/alpine_test.go | 2 +- {scan => scanner}/amazon.go | 2 +- {scan => scanner}/base.go | 34 +- {scan => scanner}/base_test.go | 2 +- {scan => scanner}/centos.go | 2 +- {scan => scanner}/debian.go | 31 +- {scan => scanner}/debian_test.go | 5 +- {scan => scanner}/executil.go | 186 +--- {scan => scanner}/executil_test.go | 2 +- {scan => scanner}/freebsd.go | 9 +- {scan => scanner}/freebsd_test.go | 2 +- {scan => scanner}/library.go | 2 +- {scan => scanner}/oracle.go | 2 +- {scan => scanner}/pseudo.go | 7 +- {scan => scanner}/redhatbase.go | 45 +- {scan => scanner}/redhatbase_test.go | 5 +- {scan => scanner}/rhel.go | 2 +- {scan => scanner}/serverapi.go | 913 ++++++++---------- {scan => scanner}/serverapi_test.go | 7 +- {scan => scanner}/suse.go | 13 +- {scan => scanner}/suse_test.go | 9 +- {scan => scanner}/unknownDistro.go | 2 +- scanner/utils.go | 96 ++ {scan => scanner}/utils_test.go | 15 +- server/empty.go | 1 - server/server.go | 27 +- subcmds/configtest.go | 40 +- subcmds/history.go | 4 +- subcmds/report.go | 204 ++-- subcmds/saas.go | 16 +- subcmds/scan.go | 48 +- subcmds/server.go | 19 +- subcmds/tui.go | 16 +- {report => tui}/tui.go | 10 +- 91 files changed, 1759 insertions(+), 1987 deletions(-) create mode 100644 config/awsconf.go create mode 100644 config/azureconf.go create mode 100644 constant/constant.go rename {report => detector}/cve_client.go (99%) rename {report => detector}/db_client.go (96%) rename report/report.go => detector/detector.go (87%) rename {github => detector}/github.go (97%) rename libmanager/libManager.go => detector/library.go (90%) create mode 100644 detector/util.go rename {wordpress => detector}/wordpress.go (98%) rename {wordpress => detector}/wordpress_test.go (98%) rename {report => reporter}/azureblob.go (80%) rename {report => reporter}/chatwork.go (96%) rename {report => reporter}/email.go (90%) rename {report => reporter}/http.go (98%) rename {report => reporter}/localfile.go (66%) rename {report => reporter}/s3.go (84%) rename {report => reporter}/slack.go (87%) rename {report => reporter}/slack_test.go (83%) rename {report => reporter}/stdout.go (73%) rename {report => reporter}/syslog.go (99%) rename {report => reporter}/syslog_test.go (99%) rename {report => reporter}/telegram.go (95%) rename {report => reporter}/util.go (86%) rename {report => reporter}/util_test.go (87%) rename {report => reporter}/writer.go (96%) delete mode 100644 scan/utils.go rename {scan => scanner}/alpine.go (97%) rename {scan => scanner}/alpine_test.go (98%) rename {scan => scanner}/amazon.go (99%) rename {scan => scanner}/base.go (97%) rename {scan => scanner}/base_test.go (99%) rename {scan => scanner}/centos.go (99%) rename {scan => scanner}/debian.go (98%) rename {scan => scanner}/debian_test.go (99%) rename {scan => scanner}/executil.go (62%) rename {scan => scanner}/executil_test.go (99%) rename {scan => scanner}/freebsd.go (97%) rename {scan => scanner}/freebsd_test.go (99%) rename {scan => scanner}/library.go (97%) rename {scan => scanner}/oracle.go (99%) rename {scan => scanner}/pseudo.go (88%) rename {scan => scanner}/redhatbase.go (95%) rename {scan => scanner}/redhatbase_test.go (99%) rename {scan => scanner}/rhel.go (99%) rename {scan => scanner}/serverapi.go (63%) rename {scan => scanner}/serverapi_test.go (95%) rename {scan => scanner}/suse.go (95%) rename {scan => scanner}/suse_test.go (95%) rename {scan => scanner}/unknownDistro.go (97%) create mode 100644 scanner/utils.go rename {scan => scanner}/utils_test.go (84%) delete mode 100644 server/empty.go rename {report => tui}/tui.go (99%) diff --git a/config/awsconf.go b/config/awsconf.go new file mode 100644 index 00000000..37ce5b69 --- /dev/null +++ b/config/awsconf.go @@ -0,0 +1,30 @@ +package config + +// AWSConf is aws config +type AWSConf struct { + // AWS profile to use + Profile string `json:"profile"` + + // AWS region to use + Region string `json:"region"` + + // S3 bucket name + S3Bucket string `json:"s3Bucket"` + + // /bucket/path/to/results + S3ResultsDir string `json:"s3ResultsDir"` + + // The Server-side encryption algorithm used when storing the reports in S3 (e.g., AES256, aws:kms). + S3ServerSideEncryption string `json:"s3ServerSideEncryption"` + + Enabled bool `toml:"-" json:"-"` +} + +// Validate configuration +func (c *AWSConf) Validate() (errs []error) { + // TODO + if !c.Enabled { + return + } + return +} diff --git a/config/azureconf.go b/config/azureconf.go new file mode 100644 index 00000000..bac1e34a --- /dev/null +++ b/config/azureconf.go @@ -0,0 +1,24 @@ +package config + +// AzureConf is azure config +type AzureConf struct { + // Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified + AccountName string `json:"accountName"` + + // Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified + AccountKey string `json:"-"` + + // Azure storage container name + ContainerName string `json:"containerName"` + + Enabled bool `toml:"-" json:"-"` +} + +// Validate configuration +func (c *AzureConf) Validate() (errs []error) { + // TODO + if !c.Enabled { + return + } + return +} diff --git a/config/chatworkconf.go b/config/chatworkconf.go index 2d0155c8..8c0fc5eb 100644 --- a/config/chatworkconf.go +++ b/config/chatworkconf.go @@ -9,11 +9,12 @@ import ( type ChatWorkConf struct { APIToken string `json:"-"` Room string `json:"-"` + Enabled bool `toml:"-" json:"-"` } // Validate validates configuration func (c *ChatWorkConf) Validate() (errs []error) { - if !Conf.ToChatWork { + if !c.Enabled { return } if len(c.Room) == 0 { diff --git a/config/config.go b/config/config.go index 771264e7..98bc90d5 100644 --- a/config/config.go +++ b/config/config.go @@ -3,11 +3,11 @@ package config import ( "fmt" "os" - "runtime" "strconv" "strings" "github.com/asaskevich/govalidator" + "github.com/future-architect/vuls/constant" log "github.com/sirupsen/logrus" "golang.org/x/xerrors" ) @@ -23,29 +23,21 @@ var Conf Config //Config is struct of Configuration type Config struct { + // scan, report Debug bool `json:"debug,omitempty"` DebugSQL bool `json:"debugSQL,omitempty"` - Lang string `json:"lang,omitempty"` HTTPProxy string `valid:"url" json:"httpProxy,omitempty"` LogDir string `json:"logDir,omitempty"` ResultsDir string `json:"resultsDir,omitempty"` Pipe bool `json:"pipe,omitempty"` Quiet bool `json:"quiet,omitempty"` - NoProgress bool `json:"noProgress,omitempty"` - SSHNative bool `json:"sshNative,omitempty"` - Vvv bool `json:"vvv,omitempty"` - Default ServerInfo `json:"default,omitempty"` - Servers map[string]ServerInfo `json:"servers,omitempty"` - CvssScoreOver float64 `json:"cvssScoreOver,omitempty"` + Default ServerInfo `json:"default,omitempty"` + Servers map[string]ServerInfo `json:"servers,omitempty"` - IgnoreUnscoredCves bool `json:"ignoreUnscoredCves,omitempty"` - IgnoreUnfixed bool `json:"ignoreUnfixed,omitempty"` - IgnoreGitHubDismissed bool `json:"ignore_git_hub_dismissed,omitempty"` - - CacheDBPath string `json:"cacheDBPath,omitempty"` - TrivyCacheDBDir string `json:"trivyCacheDBDir,omitempty"` + ScanOpts + // report CveDict GoCveDictConf `json:"cveDict,omitempty"` OvalDict GovalDictConf `json:"ovalDict,omitempty"` Gost GostConf `json:"gost,omitempty"` @@ -60,62 +52,51 @@ type Config struct { Azure AzureConf `json:"-"` ChatWork ChatWorkConf `json:"-"` Telegram TelegramConf `json:"-"` + WpScan WpScanConf `json:"-"` + Saas SaasConf `json:"-"` - WpScan WpScanConf `json:"WpScan,omitempty"` + ReportOpts +} - Saas SaasConf `json:"-"` - DetectIPS bool `json:"detectIps,omitempty"` +// ScanOpts is options for scan +type ScanOpts struct { + Vvv bool `json:"vvv,omitempty"` + DetectIPS bool `json:"detectIps,omitempty"` +} - RefreshCve bool `json:"refreshCve,omitempty"` - ToSlack bool `json:"toSlack,omitempty"` - ToChatWork bool `json:"toChatWork,omitempty"` - ToTelegram bool `json:"ToTelegram,omitempty"` - ToEmail bool `json:"toEmail,omitempty"` - ToSyslog bool `json:"toSyslog,omitempty"` - ToLocalFile bool `json:"toLocalFile,omitempty"` - ToS3 bool `json:"toS3,omitempty"` - ToAzureBlob bool `json:"toAzureBlob,omitempty"` - ToHTTP bool `json:"toHTTP,omitempty"` - FormatJSON bool `json:"formatJSON,omitempty"` - FormatOneEMail bool `json:"formatOneEMail,omitempty"` - FormatOneLineText bool `json:"formatOneLineText,omitempty"` - FormatList bool `json:"formatList,omitempty"` - FormatFullText bool `json:"formatFullText,omitempty"` - FormatCsvList bool `json:"formatCsvList,omitempty"` - GZIP bool `json:"gzip,omitempty"` - DiffPlus bool `json:"diffPlus,omitempty"` - DiffMinus bool `json:"diffMinus,omitempty"` - Diff bool `json:"diff,omitempty"` +// ReportOpts is options for report +type ReportOpts struct { + // refactored + CvssScoreOver float64 `json:"cvssScoreOver,omitempty"` + TrivyCacheDBDir string `json:"trivyCacheDBDir,omitempty"` + NoProgress bool `json:"noProgress,omitempty"` + RefreshCve bool `json:"refreshCve,omitempty"` + IgnoreUnfixed bool `json:"ignoreUnfixed,omitempty"` + IgnoreUnscoredCves bool `json:"ignoreUnscoredCves,omitempty"` + DiffPlus bool `json:"diffPlus,omitempty"` + DiffMinus bool `json:"diffMinus,omitempty"` + Diff bool `json:"diff,omitempty"` + Lang string `json:"lang,omitempty"` + + //TODO move to GitHubConf + IgnoreGitHubDismissed bool `json:"ignore_git_hub_dismissed,omitempty"` } // ValidateOnConfigtest validates func (c Config) ValidateOnConfigtest() bool { errs := c.checkSSHKeyExist() - - if runtime.GOOS == "windows" && !c.SSHNative { - errs = append(errs, xerrors.New("-ssh-native-insecure is needed on windows")) - } - - _, err := govalidator.ValidateStruct(c) - if err != nil { + if _, err := govalidator.ValidateStruct(c); err != nil { errs = append(errs, err) } - for _, err := range errs { log.Error(err) } - return len(errs) == 0 } // ValidateOnScan validates configuration func (c Config) ValidateOnScan() bool { errs := c.checkSSHKeyExist() - - if runtime.GOOS == "windows" && !c.SSHNative { - errs = append(errs, xerrors.New("-ssh-native-insecure is needed on windows")) - } - if len(c.ResultsDir) != 0 { if ok, _ := govalidator.IsFilePath(c.ResultsDir); !ok { errs = append(errs, xerrors.Errorf( @@ -123,29 +104,18 @@ func (c Config) ValidateOnScan() bool { } } - if len(c.CacheDBPath) != 0 { - if ok, _ := govalidator.IsFilePath(c.CacheDBPath); !ok { - errs = append(errs, xerrors.Errorf( - "Cache DB path must be a *Absolute* file path. -cache-dbpath: %s", - c.CacheDBPath)) - } - } - - _, err := govalidator.ValidateStruct(c) - if err != nil { + if _, err := govalidator.ValidateStruct(c); err != nil { errs = append(errs, err) } - for _, err := range errs { log.Error(err) } - return len(errs) == 0 } func (c Config) checkSSHKeyExist() (errs []error) { for serverName, v := range c.Servers { - if v.Type == ServerTypePseudo { + if v.Type == constant.ServerTypePseudo { continue } if v.KeyPath != "" { @@ -205,28 +175,37 @@ func (c Config) ValidateOnReport() bool { errs = append(errs, err) } - if mailerrs := c.EMail.Validate(); 0 < len(mailerrs) { - errs = append(errs, mailerrs...) + //TODO refactor interface + if es := c.EMail.Validate(); 0 < len(es) { + errs = append(errs, es...) } - if slackerrs := c.Slack.Validate(); 0 < len(slackerrs) { - errs = append(errs, slackerrs...) + if es := c.Slack.Validate(); 0 < len(es) { + errs = append(errs, es...) } - if chatworkerrs := c.ChatWork.Validate(); 0 < len(chatworkerrs) { - errs = append(errs, chatworkerrs...) + if es := c.ChatWork.Validate(); 0 < len(es) { + errs = append(errs, es...) } - if telegramerrs := c.Telegram.Validate(); 0 < len(telegramerrs) { - errs = append(errs, telegramerrs...) + if es := c.Telegram.Validate(); 0 < len(es) { + errs = append(errs, es...) } - if syslogerrs := c.Syslog.Validate(); 0 < len(syslogerrs) { - errs = append(errs, syslogerrs...) + if es := c.Syslog.Validate(); 0 < len(es) { + errs = append(errs, es...) } - if httperrs := c.HTTP.Validate(); 0 < len(httperrs) { - errs = append(errs, httperrs...) + if es := c.HTTP.Validate(); 0 < len(es) { + errs = append(errs, es...) + } + + if es := c.AWS.Validate(); 0 < len(es) { + errs = append(errs, es...) + } + + if es := c.Azure.Validate(); 0 < len(es) { + errs = append(errs, es...) } for _, err := range errs { @@ -309,36 +288,6 @@ func validateDB(dictionaryDBName, dbType, dbPath, dbURL string) error { return nil } -// AWSConf is aws config -type AWSConf struct { - // AWS profile to use - Profile string `json:"profile"` - - // AWS region to use - Region string `json:"region"` - - // S3 bucket name - S3Bucket string `json:"s3Bucket"` - - // /bucket/path/to/results - S3ResultsDir string `json:"s3ResultsDir"` - - // The Server-side encryption algorithm used when storing the reports in S3 (e.g., AES256, aws:kms). - S3ServerSideEncryption string `json:"s3ServerSideEncryption"` -} - -// AzureConf is azure config -type AzureConf struct { - // Azure account name to use. AZURE_STORAGE_ACCOUNT environment variable is used if not specified - AccountName string `json:"accountName"` - - // Azure account key to use. AZURE_STORAGE_ACCESS_KEY environment variable is used if not specified - AccountKey string `json:"-"` - - // Azure storage container name - ContainerName string `json:"containerName"` -} - // WpScanConf is wpscan.com config type WpScanConf struct { Token string `toml:"token,omitempty" json:"-"` @@ -354,7 +303,6 @@ type ServerInfo struct { Port string `toml:"port,omitempty" json:"port,omitempty"` SSHConfigPath string `toml:"sshConfigPath,omitempty" json:"sshConfigPath,omitempty"` KeyPath string `toml:"keyPath,omitempty" json:"keyPath,omitempty"` - KeyPassword string `json:"-" toml:"-"` CpeNames []string `toml:"cpeNames,omitempty" json:"cpeNames,omitempty"` ScanMode []string `toml:"scanMode,omitempty" json:"scanMode,omitempty"` ScanModules []string `toml:"scanModules,omitempty" json:"scanModules,omitempty"` @@ -377,7 +325,7 @@ type ServerInfo struct { IgnoredJSONKeys []string `toml:"ignoredJSONKeys,omitempty" json:"ignoredJSONKeys,omitempty"` IPv4Addrs []string `toml:"-" json:"ipv4Addrs,omitempty"` IPv6Addrs []string `toml:"-" json:"ipv6Addrs,omitempty"` - IPSIdentifiers map[IPS]string `toml:"-" json:"ipsIdentifiers,omitempty"` + IPSIdentifiers map[string]string `toml:"-" json:"ipsIdentifiers,omitempty"` WordPress *WordPressConf `toml:"wordpress,omitempty" json:"wordpress,omitempty"` // internal use @@ -434,7 +382,7 @@ func (l Distro) String() string { // MajorVersion returns Major version func (l Distro) MajorVersion() (int, error) { - if l.Family == Amazon { + if l.Family == constant.Amazon { if isAmazonLinux1(l.Release) { return 1, nil } diff --git a/config/config_test.go b/config/config_test.go index 94ec84e6..22ac7d6c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2,6 +2,8 @@ package config import ( "testing" + + . "github.com/future-architect/vuls/constant" ) func TestSyslogConfValidate(t *testing.T) { @@ -55,7 +57,7 @@ func TestSyslogConfValidate(t *testing.T) { } for i, tt := range tests { - Conf.ToSyslog = true + tt.conf.Enabled = true errs := tt.conf.Validate() if len(errs) != tt.expectedErrLength { t.Errorf("test: %d, expected %d, actual %d", i, tt.expectedErrLength, len(errs)) diff --git a/config/exploitconf.go b/config/exploitconf.go index 56633424..8a257ad2 100644 --- a/config/exploitconf.go +++ b/config/exploitconf.go @@ -54,7 +54,7 @@ func (cnf *ExploitConf) Init() { // IsFetchViaHTTP returns wether fetch via http func (cnf *ExploitConf) IsFetchViaHTTP() bool { - return Conf.Exploit.Type == "http" + return cnf.Type == "http" } // CheckHTTPHealth do health check diff --git a/config/gocvedictconf.go b/config/gocvedictconf.go index ef1c9ee7..7e6bc923 100644 --- a/config/gocvedictconf.go +++ b/config/gocvedictconf.go @@ -54,7 +54,7 @@ func (cnf *GoCveDictConf) Init() { // IsFetchViaHTTP returns wether fetch via http func (cnf *GoCveDictConf) IsFetchViaHTTP() bool { - return Conf.CveDict.Type == "http" + return cnf.Type == "http" } // CheckHTTPHealth checks http server status diff --git a/config/gostconf.go b/config/gostconf.go index 9ab07d9f..d2ec1ae3 100644 --- a/config/gostconf.go +++ b/config/gostconf.go @@ -54,7 +54,7 @@ func (cnf *GostConf) Init() { // IsFetchViaHTTP returns wether fetch via http func (cnf *GostConf) IsFetchViaHTTP() bool { - return Conf.Gost.Type == "http" + return cnf.Type == "http" } // CheckHTTPHealth do health check diff --git a/config/govaldictconf.go b/config/govaldictconf.go index 4262d4c2..622545cd 100644 --- a/config/govaldictconf.go +++ b/config/govaldictconf.go @@ -55,7 +55,7 @@ func (cnf *GovalDictConf) Init() { // IsFetchViaHTTP returns wether fetch via http func (cnf *GovalDictConf) IsFetchViaHTTP() bool { - return Conf.OvalDict.Type == "http" + return cnf.Type == "http" } // CheckHTTPHealth do health check diff --git a/config/httpconf.go b/config/httpconf.go index ca1636fa..008dd45e 100644 --- a/config/httpconf.go +++ b/config/httpconf.go @@ -8,12 +8,13 @@ import ( // HTTPConf is HTTP config type HTTPConf struct { - URL string `valid:"url" json:"-"` + URL string `valid:"url" json:"-"` + Enabled bool `toml:"-" json:"-"` } // Validate validates configuration func (c *HTTPConf) Validate() (errs []error) { - if !Conf.ToHTTP { + if !c.Enabled { return nil } @@ -28,11 +29,11 @@ const httpKey = "VULS_HTTP_URL" // Init set options with the following priority. // 1. Environment variable // 2. config.toml -func (c *HTTPConf) Init(toml HTTPConf) { +func (c *HTTPConf) Init(url string) { if os.Getenv(httpKey) != "" { c.URL = os.Getenv(httpKey) } - if toml.URL != "" { - c.URL = toml.URL + if url != "" { + c.URL = url } } diff --git a/config/ips.go b/config/ips.go index 0483e7b4..075079dc 100644 --- a/config/ips.go +++ b/config/ips.go @@ -1,9 +1,6 @@ package config -// IPS is -type IPS string - const ( // DeepSecurity is - DeepSecurity IPS = "deepsecurity" + DeepSecurity string = "deepsecurity" ) diff --git a/config/metasploitconf.go b/config/metasploitconf.go index 2638ef7e..dd19e348 100644 --- a/config/metasploitconf.go +++ b/config/metasploitconf.go @@ -53,7 +53,7 @@ func (cnf *MetasploitConf) Init() { // IsFetchViaHTTP returns wether fetch via http func (cnf *MetasploitConf) IsFetchViaHTTP() bool { - return Conf.Metasploit.Type == "http" + return cnf.Type == "http" } // CheckHTTPHealth do health check diff --git a/config/os.go b/config/os.go index 30b7f5d6..31ee5840 100644 --- a/config/os.go +++ b/config/os.go @@ -4,59 +4,8 @@ import ( "fmt" "strings" "time" -) -const ( - // RedHat is - RedHat = "redhat" - - // Debian is - Debian = "debian" - - // Ubuntu is - Ubuntu = "ubuntu" - - // CentOS is - CentOS = "centos" - - // Fedora is - // Fedora = "fedora" - - // Amazon is - Amazon = "amazon" - - // Oracle is - Oracle = "oracle" - - // FreeBSD is - FreeBSD = "freebsd" - - // Raspbian is - Raspbian = "raspbian" - - // Windows is - Windows = "windows" - - // OpenSUSE is - OpenSUSE = "opensuse" - - // OpenSUSELeap is - OpenSUSELeap = "opensuse.leap" - - // SUSEEnterpriseServer is - SUSEEnterpriseServer = "suse.linux.enterprise.server" - - // SUSEEnterpriseDesktop is - SUSEEnterpriseDesktop = "suse.linux.enterprise.desktop" - - // SUSEOpenstackCloud is - SUSEOpenstackCloud = "suse.openstack.cloud" - - // Alpine is - Alpine = "alpine" - - // ServerTypePseudo is used for ServerInfo.Type, r.Family - ServerTypePseudo = "pseudo" + "github.com/future-architect/vuls/constant" ) // EOL has End-of-Life information @@ -89,7 +38,7 @@ func (e EOL) IsExtendedSuppportEnded(now time.Time) bool { // https://github.com/aquasecurity/trivy/blob/master/pkg/detector/ospkg/redhat/redhat.go#L20 func GetEOL(family, release string) (eol EOL, found bool) { switch family { - case Amazon: + case constant.Amazon: rel := "2" if isAmazonLinux1(release) { rel = "1" @@ -98,7 +47,7 @@ func GetEOL(family, release string) (eol EOL, found bool) { "1": {StandardSupportUntil: time.Date(2023, 6, 30, 23, 59, 59, 0, time.UTC)}, "2": {}, }[rel] - case RedHat: + case constant.RedHat: // https://access.redhat.com/support/policy/updates/errata eol, found = map[string]EOL{ "3": {Ended: true}, @@ -115,7 +64,7 @@ func GetEOL(family, release string) (eol EOL, found bool) { StandardSupportUntil: time.Date(2029, 5, 31, 23, 59, 59, 0, time.UTC), }, }[major(release)] - case CentOS: + case constant.CentOS: // https://en.wikipedia.org/wiki/CentOS#End-of-support_schedule // TODO Stream eol, found = map[string]EOL{ @@ -126,7 +75,7 @@ func GetEOL(family, release string) (eol EOL, found bool) { "7": {StandardSupportUntil: time.Date(2024, 6, 30, 23, 59, 59, 0, time.UTC)}, "8": {StandardSupportUntil: time.Date(2021, 12, 31, 23, 59, 59, 0, time.UTC)}, }[major(release)] - case Oracle: + case constant.Oracle: eol, found = map[string]EOL{ // Source: // https://www.oracle.com/a/ocom/docs/elsp-lifetime-069338.pdf @@ -145,7 +94,7 @@ func GetEOL(family, release string) (eol EOL, found bool) { StandardSupportUntil: time.Date(2029, 7, 1, 23, 59, 59, 0, time.UTC), }, }[major(release)] - case Debian: + case constant.Debian: eol, found = map[string]EOL{ // https://wiki.debian.org/LTS "6": {Ended: true}, @@ -154,10 +103,10 @@ func GetEOL(family, release string) (eol EOL, found bool) { "9": {StandardSupportUntil: time.Date(2022, 6, 30, 23, 59, 59, 0, time.UTC)}, "10": {StandardSupportUntil: time.Date(2024, 6, 30, 23, 59, 59, 0, time.UTC)}, }[major(release)] - case Raspbian: + case constant.Raspbian: // Not found eol, found = map[string]EOL{}[major(release)] - case Ubuntu: + case constant.Ubuntu: // https://wiki.ubuntu.com/Releases eol, found = map[string]EOL{ "14.10": {Ended: true}, @@ -189,9 +138,9 @@ func GetEOL(family, release string) (eol EOL, found bool) { StandardSupportUntil: time.Date(2022, 7, 1, 23, 59, 59, 0, time.UTC), }, }[release] - case SUSEEnterpriseServer: + case constant.SUSEEnterpriseServer: //TODO - case Alpine: + case constant.Alpine: // https://github.com/aquasecurity/trivy/blob/master/pkg/detector/ospkg/alpine/alpine.go#L19 // https://wiki.alpinelinux.org/wiki/Alpine_Linux:Releases eol, found = map[string]EOL{ @@ -218,7 +167,7 @@ func GetEOL(family, release string) (eol EOL, found bool) { "3.12": {StandardSupportUntil: time.Date(2022, 5, 1, 23, 59, 59, 0, time.UTC)}, "3.13": {StandardSupportUntil: time.Date(2022, 11, 1, 23, 59, 59, 0, time.UTC)}, }[majorDotMinor(release)] - case FreeBSD: + case constant.FreeBSD: // https://www.freebsd.org/security/ eol, found = map[string]EOL{ "7": {Ended: true}, diff --git a/config/os_test.go b/config/os_test.go index 87ea93a3..682a7f34 100644 --- a/config/os_test.go +++ b/config/os_test.go @@ -3,6 +3,8 @@ package config import ( "testing" "time" + + . "github.com/future-architect/vuls/constant" ) func TestEOL_IsStandardSupportEnded(t *testing.T) { diff --git a/config/slackconf.go b/config/slackconf.go index affa9ce5..074eefe5 100644 --- a/config/slackconf.go +++ b/config/slackconf.go @@ -16,11 +16,12 @@ type SlackConf struct { AuthUser string `json:"-" toml:"authUser,omitempty"` NotifyUsers []string `toml:"notifyUsers,omitempty" json:"-"` Text string `json:"-"` + Enabled bool `toml:"-" json:"-"` } // Validate validates configuration func (c *SlackConf) Validate() (errs []error) { - if !Conf.ToSlack { + if !c.Enabled { return } diff --git a/config/smtpconf.go b/config/smtpconf.go index 43e85332..9b5f6189 100644 --- a/config/smtpconf.go +++ b/config/smtpconf.go @@ -15,6 +15,7 @@ type SMTPConf struct { To []string `toml:"to,omitempty" json:"-"` Cc []string `toml:"cc,omitempty" json:"-"` SubjectPrefix string `toml:"subjectPrefix,omitempty" json:"-"` + Enabled bool `toml:"-" json:"-"` } func checkEmails(emails []string) (errs []error) { @@ -31,10 +32,9 @@ func checkEmails(emails []string) (errs []error) { // Validate SMTP configuration func (c *SMTPConf) Validate() (errs []error) { - if !Conf.ToEmail { + if !c.Enabled { return } - // Check Emails fromat emails := []string{} emails = append(emails, c.From) emails = append(emails, c.To...) @@ -44,10 +44,10 @@ func (c *SMTPConf) Validate() (errs []error) { errs = append(errs, emailErrs...) } - if len(c.SMTPAddr) == 0 { + if c.SMTPAddr == "" { errs = append(errs, xerrors.New("email.smtpAddr must not be empty")) } - if len(c.SMTPPort) == 0 { + if c.SMTPPort == "" { errs = append(errs, xerrors.New("email.smtpPort must not be empty")) } if len(c.To) == 0 { diff --git a/config/syslogconf.go b/config/syslogconf.go index 6e8fc614..d68fe70f 100644 --- a/config/syslogconf.go +++ b/config/syslogconf.go @@ -17,11 +17,12 @@ type SyslogConf struct { Facility string `json:"-"` Tag string `json:"-"` Verbose bool `json:"-"` + Enabled bool `toml:"-" json:"-"` } // Validate validates configuration func (c *SyslogConf) Validate() (errs []error) { - if !Conf.ToSyslog { + if !c.Enabled { return nil } // If protocol is empty, it will connect to the local syslog server. diff --git a/config/telegramconf.go b/config/telegramconf.go index d15f660b..349a7067 100644 --- a/config/telegramconf.go +++ b/config/telegramconf.go @@ -7,13 +7,14 @@ import ( // TelegramConf is Telegram config type TelegramConf struct { - Token string `json:"-"` - ChatID string `json:"-"` + Token string `json:"-"` + ChatID string `json:"-"` + Enabled bool `toml:"-" json:"-"` } // Validate validates configuration func (c *TelegramConf) Validate() (errs []error) { - if !Conf.ToTelegram { + if !c.Enabled { return } if len(c.ChatID) == 0 { diff --git a/config/tomlloader.go b/config/tomlloader.go index 44b7f62d..f2b73ef7 100644 --- a/config/tomlloader.go +++ b/config/tomlloader.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/BurntSushi/toml" + "github.com/future-architect/vuls/constant" "github.com/knqyf263/go-cpe/naming" "golang.org/x/xerrors" ) @@ -18,10 +19,6 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error { if _, err := toml.DecodeFile(pathToToml, &Conf); err != nil { return err } - if keyPass != "" { - Conf.Default.KeyPassword = keyPass - } - Conf.CveDict.Init() Conf.OvalDict.Init() Conf.Gost.Init() @@ -31,10 +28,6 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error { index := 0 for name, server := range Conf.Servers { server.ServerName = name - if 0 < len(server.KeyPassword) { - return xerrors.Errorf("[Deprecated] KEYPASSWORD IN CONFIG FILE ARE UNSECURE. REMOVE THEM IMMEDIATELY FOR A SECURITY REASONS. THEY WILL BE REMOVED IN A FUTURE RELEASE: %s", name) - } - if err := setDefaultIfEmpty(&server, Conf.Default); err != nil { return xerrors.Errorf("Failed to set default value to config. server: %s, err: %w", name, err) } @@ -135,7 +128,7 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error { } func setDefaultIfEmpty(server *ServerInfo, d ServerInfo) error { - if server.Type != ServerTypePseudo { + if server.Type != constant.ServerTypePseudo { if len(server.Host) == 0 { return xerrors.Errorf("server.host is empty") } @@ -166,10 +159,6 @@ func setDefaultIfEmpty(server *ServerInfo, d ServerInfo) error { if server.KeyPath == "" { server.KeyPath = Conf.Default.KeyPath } - - if server.KeyPassword == "" { - server.KeyPassword = Conf.Default.KeyPassword - } } if len(server.Lockfiles) == 0 { diff --git a/constant/constant.go b/constant/constant.go new file mode 100644 index 00000000..8a55d1c1 --- /dev/null +++ b/constant/constant.go @@ -0,0 +1,58 @@ +package constant + +// Global constant +// Pkg local constants should not be defined here. +// Define them in the each package. + +const ( + // RedHat is + RedHat = "redhat" + + // Debian is + Debian = "debian" + + // Ubuntu is + Ubuntu = "ubuntu" + + // CentOS is + CentOS = "centos" + + // Fedora is + // Fedora = "fedora" + + // Amazon is + Amazon = "amazon" + + // Oracle is + Oracle = "oracle" + + // FreeBSD is + FreeBSD = "freebsd" + + // Raspbian is + Raspbian = "raspbian" + + // Windows is + Windows = "windows" + + // OpenSUSE is + OpenSUSE = "opensuse" + + // OpenSUSELeap is + OpenSUSELeap = "opensuse.leap" + + // SUSEEnterpriseServer is + SUSEEnterpriseServer = "suse.linux.enterprise.server" + + // SUSEEnterpriseDesktop is + SUSEEnterpriseDesktop = "suse.linux.enterprise.desktop" + + // SUSEOpenstackCloud is + SUSEOpenstackCloud = "suse.openstack.cloud" + + // Alpine is + Alpine = "alpine" + + // ServerTypePseudo is used for ServerInfo.Type, r.Family + ServerTypePseudo = "pseudo" +) diff --git a/report/cve_client.go b/detector/cve_client.go similarity index 99% rename from report/cve_client.go rename to detector/cve_client.go index 3043cd7d..0570ca38 100644 --- a/report/cve_client.go +++ b/detector/cve_client.go @@ -1,6 +1,6 @@ // +build !scanner -package report +package detector import ( "encoding/json" diff --git a/report/db_client.go b/detector/db_client.go similarity index 96% rename from report/db_client.go rename to detector/db_client.go index 334ce87f..4abb437b 100644 --- a/report/db_client.go +++ b/detector/db_client.go @@ -1,6 +1,6 @@ // +build !scanner -package report +package detector import ( "os" @@ -91,7 +91,7 @@ func NewDBClient(cnf DBClientConf) (dbclient *DBClient, locked bool, err error) // NewCveDB returns cve db client func NewCveDB(cnf DBClientConf) (driver cvedb.DB, locked bool, err error) { - if config.Conf.CveDict.IsFetchViaHTTP() { + if cnf.CveDictCnf.IsFetchViaHTTP() { return nil, false, nil } util.Log.Debugf("open cve-dictionary db (%s)", cnf.CveDictCnf.Type) @@ -115,7 +115,7 @@ func NewCveDB(cnf DBClientConf) (driver cvedb.DB, locked bool, err error) { // NewOvalDB returns oval db client func NewOvalDB(cnf DBClientConf) (driver ovaldb.DB, locked bool, err error) { - if config.Conf.OvalDict.IsFetchViaHTTP() { + if cnf.OvalDictCnf.IsFetchViaHTTP() { return nil, false, nil } path := cnf.OvalDictCnf.URL @@ -142,7 +142,7 @@ func NewOvalDB(cnf DBClientConf) (driver ovaldb.DB, locked bool, err error) { // NewGostDB returns db client for Gost func NewGostDB(cnf DBClientConf) (driver gostdb.DB, locked bool, err error) { - if config.Conf.Gost.IsFetchViaHTTP() { + if cnf.GostCnf.IsFetchViaHTTP() { return nil, false, nil } path := cnf.GostCnf.URL @@ -168,7 +168,7 @@ func NewGostDB(cnf DBClientConf) (driver gostdb.DB, locked bool, err error) { // NewExploitDB returns db client for Exploit func NewExploitDB(cnf DBClientConf) (driver exploitdb.DB, locked bool, err error) { - if config.Conf.Exploit.IsFetchViaHTTP() { + if cnf.ExploitCnf.IsFetchViaHTTP() { return nil, false, nil } path := cnf.ExploitCnf.URL @@ -194,7 +194,7 @@ func NewExploitDB(cnf DBClientConf) (driver exploitdb.DB, locked bool, err error // NewMetasploitDB returns db client for Metasploit func NewMetasploitDB(cnf DBClientConf) (driver metasploitdb.DB, locked bool, err error) { - if config.Conf.Metasploit.IsFetchViaHTTP() { + if cnf.MetasploitCnf.IsFetchViaHTTP() { return nil, false, nil } path := cnf.MetasploitCnf.URL diff --git a/report/report.go b/detector/detector.go similarity index 87% rename from report/report.go rename to detector/detector.go index 694f8e7e..8666de90 100644 --- a/report/report.go +++ b/detector/detector.go @@ -1,6 +1,6 @@ // +build !scanner -package report +package detector import ( "os" @@ -9,17 +9,16 @@ import ( "github.com/future-architect/vuls/config" c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/contrib/owasp-dependency-check/parser" "github.com/future-architect/vuls/cwe" "github.com/future-architect/vuls/exploit" - "github.com/future-architect/vuls/github" "github.com/future-architect/vuls/gost" - "github.com/future-architect/vuls/libmanager" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/msf" "github.com/future-architect/vuls/oval" + "github.com/future-architect/vuls/reporter" "github.com/future-architect/vuls/util" - "github.com/future-architect/vuls/wordpress" gostdb "github.com/knqyf263/gost/db" cvedb "github.com/kotakanbe/go-cve-dictionary/db" cvemodels "github.com/kotakanbe/go-cve-dictionary/models" @@ -29,8 +28,12 @@ import ( "golang.org/x/xerrors" ) -// FillCveInfos fills CVE Detailed Information -func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.ScanResult, error) { +// type Detector struct { +// Targets map[string]config.ServerInfo +// } + +// Detect vulns and fill CVE detailed information +func Detect(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.ScanResult, error) { // Use the same reportedAt for all rs reportedAt := time.Now() @@ -74,7 +77,7 @@ func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]mode } } - if err := libmanager.DetectLibsCves(&r); err != nil { + if err := DetectLibsCves(&r, c.Conf.TrivyCacheDBDir, c.Conf.NoProgress); err != nil { return nil, xerrors.Errorf("Failed to fill with Library dependency: %w", err) } @@ -87,7 +90,7 @@ func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]mode } repos := c.Conf.Servers[r.ServerName].GitHubRepos - if err := DetectGitHubCves(&r, repos); err != nil { + if err := DetectGitHubCves(&r, repos, c.Conf.IgnoreGitHubDismissed); err != nil { return nil, xerrors.Errorf("Failed to detect GitHub Cves: %w", err) } @@ -116,7 +119,8 @@ func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]mode if s, ok := c.Conf.Servers[r.ServerName]; ok { r = r.ClearFields(s.IgnoredJSONKeys) } - if err := overwriteJSONFile(dir, r); err != nil { + //TODO don't call here + if err := reporter.OverwriteJSONFile(dir, r); err != nil { return nil, xerrors.Errorf("Failed to write JSON: %w", err) } } @@ -131,13 +135,32 @@ func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]mode for i, r := range rs { r = r.FilterByCvssOver(c.Conf.CvssScoreOver) - r = r.FilterIgnoreCves() r = r.FilterUnfixed(c.Conf.IgnoreUnfixed) - r = r.FilterIgnorePkgs() r = r.FilterInactiveWordPressLibs(c.Conf.WpScan.DetectInactive) + + // IgnoreCves + ignoreCves := []string{} + if r.Container.Name == "" { + ignoreCves = c.Conf.Servers[r.ServerName].IgnoreCves + } else if con, ok := c.Conf.Servers[r.ServerName].Containers[r.Container.Name]; ok { + ignoreCves = con.IgnoreCves + } + r = r.FilterIgnoreCves(ignoreCves) + + // ignorePkgs + ignorePkgsRegexps := []string{} + if r.Container.Name == "" { + ignorePkgsRegexps = config.Conf.Servers[r.ServerName].IgnorePkgsRegexp + } else if s, ok := config.Conf.Servers[r.ServerName].Containers[r.Container.Name]; ok { + ignorePkgsRegexps = s.IgnorePkgsRegexp + } + r = r.FilterIgnorePkgs(ignorePkgsRegexps) + + // IgnoreUnscored if c.Conf.IgnoreUnscoredCves { r.ScannedCves = r.ScannedCves.FindScoredVulns() } + rs[i] = r } return rs, nil @@ -158,7 +181,7 @@ func DetectPkgCves(dbclient DBClient, r *models.ScanResult) error { } } else if reuseScannedCves(r) { util.Log.Infof("r.Release is empty. Use CVEs as it as.") - } else if r.Family == c.ServerTypePseudo { + } else if r.Family == constant.ServerTypePseudo { util.Log.Infof("pseudo type. Skip OVAL and gost detection") } else { return xerrors.Errorf("Failed to fill CVEs. r.Release is empty") @@ -195,7 +218,7 @@ func DetectPkgCves(dbclient DBClient, r *models.ScanResult) error { } // DetectGitHubCves fetches CVEs from GitHub Security Alerts -func DetectGitHubCves(r *models.ScanResult, githubConfs map[string]c.GitHubConf) error { +func DetectGitHubCves(r *models.ScanResult, githubConfs map[string]c.GitHubConf, ignoreDismissed bool) error { if len(githubConfs) == 0 { return nil } @@ -205,7 +228,7 @@ func DetectGitHubCves(r *models.ScanResult, githubConfs map[string]c.GitHubConf) return xerrors.Errorf("Failed to parse GitHub owner/repo: %s", ownerRepo) } owner, repo := ss[0], ss[1] - n, err := github.DetectGitHubSecurityAlerts(r, owner, repo, setting.Token) + n, err := DetectGitHubSecurityAlerts(r, owner, repo, setting.Token, ignoreDismissed) if err != nil { return xerrors.Errorf("Failed to access GitHub Security Alerts: %w", err) } @@ -221,7 +244,7 @@ func DetectWordPressCves(r *models.ScanResult, wpCnf *c.WpScanConf) error { return nil } util.Log.Infof("Detect WordPress CVE. pkgs: %d ", len(r.WordPressPackages)) - n, err := wordpress.DetectWordPressCves(r, wpCnf) + n, err := detectWordPressCves(r, wpCnf) if err != nil { return xerrors.Errorf("Failed to detect WordPress CVE: %w", err) } @@ -328,35 +351,35 @@ func detectPkgsCvesWithOval(driver ovaldb.DB, r *models.ScanResult) error { var ovalFamily string switch r.Family { - case c.Debian, c.Raspbian: + case constant.Debian, constant.Raspbian: ovalClient = oval.NewDebian() - ovalFamily = c.Debian - case c.Ubuntu: + ovalFamily = constant.Debian + case constant.Ubuntu: ovalClient = oval.NewUbuntu() - ovalFamily = c.Ubuntu - case c.RedHat: + ovalFamily = constant.Ubuntu + case constant.RedHat: ovalClient = oval.NewRedhat() - ovalFamily = c.RedHat - case c.CentOS: + ovalFamily = constant.RedHat + case constant.CentOS: ovalClient = oval.NewCentOS() //use RedHat's OVAL - ovalFamily = c.RedHat - case c.Oracle: + ovalFamily = constant.RedHat + case constant.Oracle: ovalClient = oval.NewOracle() - ovalFamily = c.Oracle - case c.SUSEEnterpriseServer: + ovalFamily = constant.Oracle + case constant.SUSEEnterpriseServer: // TODO other suse family ovalClient = oval.NewSUSE() - ovalFamily = c.SUSEEnterpriseServer - case c.Alpine: + ovalFamily = constant.SUSEEnterpriseServer + case constant.Alpine: ovalClient = oval.NewAlpine() - ovalFamily = c.Alpine - case c.Amazon: + ovalFamily = constant.Alpine + case constant.Amazon: ovalClient = oval.NewAmazon() - ovalFamily = c.Amazon - case c.FreeBSD, c.Windows: + ovalFamily = constant.Amazon + case constant.FreeBSD, constant.Windows: return nil - case c.ServerTypePseudo: + case constant.ServerTypePseudo: return nil default: if r.Family == "" { @@ -484,7 +507,7 @@ func fillCweDict(r *models.ScanResult) { entry.En = &cwe.Cwe{CweID: id} } - if c.Conf.Lang == "ja" { + if r.Lang == "ja" { if e, ok := cwe.CweDictJa[id]; ok { if rank, ok := cwe.OwaspTopTen2017[id]; ok { entry.OwaspTopTen2017 = rank diff --git a/github/github.go b/detector/github.go similarity index 97% rename from github/github.go rename to detector/github.go index aae7b3c0..a012ba97 100644 --- a/github/github.go +++ b/detector/github.go @@ -1,4 +1,4 @@ -package github +package detector import ( "bytes" @@ -9,7 +9,6 @@ import ( "net/http" "time" - "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/errof" "github.com/future-architect/vuls/models" "golang.org/x/oauth2" @@ -18,7 +17,7 @@ import ( // DetectGitHubSecurityAlerts access to owner/repo on GitHub and fetch security alerts of the repository via GitHub API v4 GraphQL and then set to the given ScanResult. // https://help.github.com/articles/about-security-alerts-for-vulnerable-dependencies/ //TODO move to report -func DetectGitHubSecurityAlerts(r *models.ScanResult, owner, repo, token string) (nCVEs int, err error) { +func DetectGitHubSecurityAlerts(r *models.ScanResult, owner, repo, token string, ignoreDismissed bool) (nCVEs int, err error) { src := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, ) @@ -74,7 +73,7 @@ func DetectGitHubSecurityAlerts(r *models.ScanResult, owner, repo, token string) } for _, v := range alerts.Data.Repository.VulnerabilityAlerts.Edges { - if config.Conf.IgnoreGitHubDismissed && v.Node.DismissReason != "" { + if ignoreDismissed && v.Node.DismissReason != "" { continue } diff --git a/libmanager/libManager.go b/detector/library.go similarity index 90% rename from libmanager/libManager.go rename to detector/library.go index a57954fb..05f39896 100644 --- a/libmanager/libManager.go +++ b/detector/library.go @@ -1,4 +1,4 @@ -package libmanager +package detector import ( "context" @@ -12,13 +12,12 @@ import ( "golang.org/x/xerrors" "k8s.io/utils/clock" - "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" ) // DetectLibsCves fills LibraryScanner information -func DetectLibsCves(r *models.ScanResult) (err error) { +func DetectLibsCves(r *models.ScanResult, cacheDir string, noProgress bool) (err error) { totalCnt := 0 if len(r.LibraryScanners) == 0 { return @@ -31,11 +30,11 @@ func DetectLibsCves(r *models.ScanResult) (err error) { } util.Log.Info("Updating library db...") - if err := downloadDB(config.Version, config.Conf.TrivyCacheDBDir, config.Conf.NoProgress, false, false); err != nil { + if err := downloadDB("", cacheDir, noProgress, false, false); err != nil { return err } - if err := db2.Init(config.Conf.TrivyCacheDBDir); err != nil { + if err := db2.Init(cacheDir); err != nil { return err } defer db2.Close() diff --git a/detector/util.go b/detector/util.go new file mode 100644 index 00000000..550e4fb2 --- /dev/null +++ b/detector/util.go @@ -0,0 +1,270 @@ +package detector + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "sort" + "time" + + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/util" + "golang.org/x/xerrors" +) + +func reuseScannedCves(r *models.ScanResult) bool { + switch r.Family { + case constant.FreeBSD, constant.Raspbian: + return true + } + if isTrivyResult(r) { + return true + } + return false +} + +func isTrivyResult(r *models.ScanResult) bool { + _, ok := r.Optional["trivy-target"] + return ok +} + +func needToRefreshCve(r models.ScanResult) bool { + for _, cve := range r.ScannedCves { + if 0 < len(cve.CveContents) { + return false + } + } + return true +} + +func loadPrevious(currs models.ScanResults) (prevs models.ScanResults, err error) { + dirs, err := ListValidJSONDirs() + if err != nil { + return + } + + for _, result := range currs { + filename := result.ServerName + ".json" + if result.Container.Name != "" { + filename = fmt.Sprintf("%s@%s.json", result.Container.Name, result.ServerName) + } + for _, dir := range dirs[1:] { + path := filepath.Join(dir, filename) + r, err := loadOneServerScanResult(path) + if err != nil { + util.Log.Debugf("%+v", err) + continue + } + if r.Family == result.Family && r.Release == result.Release { + prevs = append(prevs, *r) + util.Log.Infof("Previous json found: %s", path) + break + } else { + util.Log.Infof("Previous json is different family.Release: %s, pre: %s.%s cur: %s.%s", + path, r.Family, r.Release, result.Family, result.Release) + } + } + } + return prevs, nil +} + +func diff(curResults, preResults models.ScanResults, isPlus, isMinus bool) (diffed models.ScanResults) { + for _, current := range curResults { + found := false + var previous models.ScanResult + for _, r := range preResults { + if current.ServerName == r.ServerName && current.Container.Name == r.Container.Name { + found = true + previous = r + break + } + } + + if !found { + diffed = append(diffed, current) + continue + } + + cves := models.VulnInfos{} + if isPlus { + cves = getPlusDiffCves(previous, current) + } + if isMinus { + minus := getMinusDiffCves(previous, current) + if len(cves) == 0 { + cves = minus + } else { + for k, v := range minus { + cves[k] = v + } + } + } + + packages := models.Packages{} + for _, s := range cves { + for _, affected := range s.AffectedPackages { + var p models.Package + if s.DiffStatus == models.DiffPlus { + p = current.Packages[affected.Name] + } else { + p = previous.Packages[affected.Name] + } + packages[affected.Name] = p + } + } + current.ScannedCves = cves + current.Packages = packages + diffed = append(diffed, current) + } + return +} + +func getPlusDiffCves(previous, current models.ScanResult) models.VulnInfos { + previousCveIDsSet := map[string]bool{} + for _, previousVulnInfo := range previous.ScannedCves { + previousCveIDsSet[previousVulnInfo.CveID] = true + } + + new := models.VulnInfos{} + updated := models.VulnInfos{} + for _, v := range current.ScannedCves { + if previousCveIDsSet[v.CveID] { + if isCveInfoUpdated(v.CveID, previous, current) { + v.DiffStatus = models.DiffPlus + updated[v.CveID] = v + util.Log.Debugf("updated: %s", v.CveID) + + // TODO commented out because a bug of diff logic when multiple oval defs found for a certain CVE-ID and same updated_at + // if these OVAL defs have different affected packages, this logic detects as updated. + // This logic will be uncomented after integration with gost https://github.com/knqyf263/gost + // } else if isCveFixed(v, previous) { + // updated[v.CveID] = v + // util.Log.Debugf("fixed: %s", v.CveID) + + } else { + util.Log.Debugf("same: %s", v.CveID) + } + } else { + util.Log.Debugf("new: %s", v.CveID) + v.DiffStatus = models.DiffPlus + new[v.CveID] = v + } + } + + if len(updated) == 0 && len(new) == 0 { + util.Log.Infof("%s: There are %d vulnerabilities, but no difference between current result and previous one.", current.FormatServerName(), len(current.ScannedCves)) + } + + for cveID, vuln := range new { + updated[cveID] = vuln + } + return updated +} + +func getMinusDiffCves(previous, current models.ScanResult) models.VulnInfos { + currentCveIDsSet := map[string]bool{} + for _, currentVulnInfo := range current.ScannedCves { + currentCveIDsSet[currentVulnInfo.CveID] = true + } + + clear := models.VulnInfos{} + for _, v := range previous.ScannedCves { + if !currentCveIDsSet[v.CveID] { + v.DiffStatus = models.DiffMinus + clear[v.CveID] = v + util.Log.Debugf("clear: %s", v.CveID) + } + } + if len(clear) == 0 { + util.Log.Infof("%s: There are %d vulnerabilities, but no difference between current result and previous one.", current.FormatServerName(), len(current.ScannedCves)) + } + + return clear +} + +func isCveInfoUpdated(cveID string, previous, current models.ScanResult) bool { + cTypes := []models.CveContentType{ + models.Nvd, + models.Jvn, + models.NewCveContentType(current.Family), + } + + prevLastModified := map[models.CveContentType]time.Time{} + preVinfo, ok := previous.ScannedCves[cveID] + if !ok { + return true + } + for _, cType := range cTypes { + if content, ok := preVinfo.CveContents[cType]; ok { + prevLastModified[cType] = content.LastModified + } + } + + curLastModified := map[models.CveContentType]time.Time{} + curVinfo, ok := current.ScannedCves[cveID] + if !ok { + return true + } + for _, cType := range cTypes { + if content, ok := curVinfo.CveContents[cType]; ok { + curLastModified[cType] = content.LastModified + } + } + + for _, t := range cTypes { + if !curLastModified[t].Equal(prevLastModified[t]) { + util.Log.Debugf("%s LastModified not equal: \n%s\n%s", + cveID, curLastModified[t], prevLastModified[t]) + return true + } + } + return false +} + +// jsonDirPattern is file name pattern of JSON directory +// 2016-11-16T10:43:28+09:00 +// 2016-11-16T10:43:28Z +var jsonDirPattern = regexp.MustCompile( + `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})$`) + +// ListValidJSONDirs returns valid json directory as array +// Returned array is sorted so that recent directories are at the head +func ListValidJSONDirs() (dirs []string, err error) { + var dirInfo []os.FileInfo + if dirInfo, err = ioutil.ReadDir(config.Conf.ResultsDir); err != nil { + err = xerrors.Errorf("Failed to read %s: %w", + config.Conf.ResultsDir, err) + return + } + for _, d := range dirInfo { + if d.IsDir() && jsonDirPattern.MatchString(d.Name()) { + jsonDir := filepath.Join(config.Conf.ResultsDir, d.Name()) + dirs = append(dirs, jsonDir) + } + } + sort.Slice(dirs, func(i, j int) bool { + return dirs[j] < dirs[i] + }) + return +} + +// loadOneServerScanResult read JSON data of one server +func loadOneServerScanResult(jsonFile string) (*models.ScanResult, error) { + var ( + data []byte + err error + ) + if data, err = ioutil.ReadFile(jsonFile); err != nil { + return nil, xerrors.Errorf("Failed to read %s: %w", jsonFile, err) + } + result := &models.ScanResult{} + if err := json.Unmarshal(data, result); err != nil { + return nil, xerrors.Errorf("Failed to parse %s: %w", jsonFile, err) + } + return result, nil +} diff --git a/wordpress/wordpress.go b/detector/wordpress.go similarity index 98% rename from wordpress/wordpress.go rename to detector/wordpress.go index 8d120c57..ceabc9ce 100644 --- a/wordpress/wordpress.go +++ b/detector/wordpress.go @@ -1,4 +1,4 @@ -package wordpress +package detector import ( "context" @@ -50,8 +50,7 @@ type References struct { // DetectWordPressCves access to wpscan and fetch scurity alerts and then set to the given ScanResult. // https://wpscan.com/ -// TODO move to report -func DetectWordPressCves(r *models.ScanResult, cnf *c.WpScanConf) (int, error) { +func detectWordPressCves(r *models.ScanResult, cnf *c.WpScanConf) (int, error) { if len(r.WordPressPackages) == 0 { return 0, nil } diff --git a/wordpress/wordpress_test.go b/detector/wordpress_test.go similarity index 98% rename from wordpress/wordpress_test.go rename to detector/wordpress_test.go index 909a0f1e..99165208 100644 --- a/wordpress/wordpress_test.go +++ b/detector/wordpress_test.go @@ -1,4 +1,4 @@ -package wordpress +package detector import ( "reflect" diff --git a/go.mod b/go.mod index baf964dd..5374d0a6 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,6 @@ require ( github.com/spf13/cobra v1.1.1 github.com/takuzoo3868/go-msfdb v0.1.3 github.com/vulsio/go-exploitdb v0.1.4 - golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad golang.org/x/oauth2 v0.0.0-20210125201302-af13f521f196 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 diff --git a/gost/debian.go b/gost/debian.go index be431347..94fc1c42 100644 --- a/gost/debian.go +++ b/gost/debian.go @@ -6,6 +6,7 @@ import ( "encoding/json" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" "github.com/knqyf263/gost/db" @@ -56,7 +57,7 @@ func (deb Debian) DetectUnfixed(driver db.DB, r *models.ScanResult, _ bool) (nCV // Debian Security Tracker does not support Package for Raspbian, so skip it. var scanResult models.ScanResult - if r.Family != config.Raspbian { + if r.Family != constant.Raspbian { scanResult = *r } else { scanResult = r.RemoveRaspbianPackFromResult() diff --git a/gost/gost.go b/gost/gost.go index aaaaa12e..d715bd2b 100644 --- a/gost/gost.go +++ b/gost/gost.go @@ -3,9 +3,10 @@ package gost import ( - cnf "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" "github.com/knqyf263/gost/db" + + "github.com/future-architect/vuls/constant" ) // Client is the interface of OVAL client. @@ -17,11 +18,11 @@ type Client interface { // NewClient make Client by family func NewClient(family string) Client { switch family { - case cnf.RedHat, cnf.CentOS: + case constant.RedHat, constant.CentOS: return RedHat{} - case cnf.Debian, cnf.Raspbian: + case constant.Debian, constant.Raspbian: return Debian{} - case cnf.Windows: + case constant.Windows: return Microsoft{} default: return Pseudo{} diff --git a/models/scanresults.go b/models/scanresults.go index a64b2e7b..c83beb41 100644 --- a/models/scanresults.go +++ b/models/scanresults.go @@ -9,6 +9,7 @@ import ( "time" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/cwe" "github.com/future-architect/vuls/util" ) @@ -18,31 +19,31 @@ type ScanResults []ScanResult // ScanResult has the result of scanned CVE information. type ScanResult struct { - JSONVersion int `json:"jsonVersion"` - Lang string `json:"lang"` - ServerUUID string `json:"serverUUID"` - ServerName string `json:"serverName"` // TOML Section key - Family string `json:"family"` - Release string `json:"release"` - Container Container `json:"container"` - Platform Platform `json:"platform"` - IPv4Addrs []string `json:"ipv4Addrs,omitempty"` // only global unicast address (https://golang.org/pkg/net/#IP.IsGlobalUnicast) - IPv6Addrs []string `json:"ipv6Addrs,omitempty"` // only global unicast address (https://golang.org/pkg/net/#IP.IsGlobalUnicast) - IPSIdentifiers map[config.IPS]string `json:"ipsIdentifiers,omitempty"` - ScannedAt time.Time `json:"scannedAt"` - ScanMode string `json:"scanMode"` - ScannedVersion string `json:"scannedVersion"` - ScannedRevision string `json:"scannedRevision"` - ScannedBy string `json:"scannedBy"` - ScannedVia string `json:"scannedVia"` - ScannedIPv4Addrs []string `json:"scannedIpv4Addrs,omitempty"` - ScannedIPv6Addrs []string `json:"scannedIpv6Addrs,omitempty"` - ReportedAt time.Time `json:"reportedAt"` - ReportedVersion string `json:"reportedVersion"` - ReportedRevision string `json:"reportedRevision"` - ReportedBy string `json:"reportedBy"` - Errors []string `json:"errors"` - Warnings []string `json:"warnings"` + JSONVersion int `json:"jsonVersion"` + Lang string `json:"lang"` + ServerUUID string `json:"serverUUID"` + ServerName string `json:"serverName"` // TOML Section key + Family string `json:"family"` + Release string `json:"release"` + Container Container `json:"container"` + Platform Platform `json:"platform"` + IPv4Addrs []string `json:"ipv4Addrs,omitempty"` // only global unicast address (https://golang.org/pkg/net/#IP.IsGlobalUnicast) + IPv6Addrs []string `json:"ipv6Addrs,omitempty"` // only global unicast address (https://golang.org/pkg/net/#IP.IsGlobalUnicast) + IPSIdentifiers map[string]string `json:"ipsIdentifiers,omitempty"` + ScannedAt time.Time `json:"scannedAt"` + ScanMode string `json:"scanMode"` + ScannedVersion string `json:"scannedVersion"` + ScannedRevision string `json:"scannedRevision"` + ScannedBy string `json:"scannedBy"` + ScannedVia string `json:"scannedVia"` + ScannedIPv4Addrs []string `json:"scannedIpv4Addrs,omitempty"` + ScannedIPv6Addrs []string `json:"scannedIpv6Addrs,omitempty"` + ReportedAt time.Time `json:"reportedAt"` + ReportedVersion string `json:"reportedVersion"` + ReportedRevision string `json:"reportedRevision"` + ReportedBy string `json:"reportedBy"` + Errors []string `json:"errors"` + Warnings []string `json:"warnings"` ScannedCves VulnInfos `json:"scannedCves"` RunningKernel Kernel `json:"runningKernel"` @@ -56,66 +57,22 @@ type ScanResult struct { Config struct { Scan config.Config `json:"scan"` Report config.Config `json:"report"` - } `json:"config"` + } `json:"constant"` } -// CweDict is a dictionary for CWE -type CweDict map[string]CweDictEntry - -// Get the name, url, top10URL for the specified cweID, lang -func (c CweDict) Get(cweID, lang string) (name, url, top10Rank, top10URL, cweTop25Rank, cweTop25URL, sansTop25Rank, sansTop25URL string) { - cweNum := strings.TrimPrefix(cweID, "CWE-") - switch config.Conf.Lang { - case "ja": - if dict, ok := c[cweNum]; ok && dict.OwaspTopTen2017 != "" { - top10Rank = dict.OwaspTopTen2017 - top10URL = cwe.OwaspTopTen2017GitHubURLJa[dict.OwaspTopTen2017] - } - if dict, ok := c[cweNum]; ok && dict.CweTopTwentyfive2019 != "" { - cweTop25Rank = dict.CweTopTwentyfive2019 - cweTop25URL = cwe.CweTopTwentyfive2019URL - } - if dict, ok := c[cweNum]; ok && dict.SansTopTwentyfive != "" { - sansTop25Rank = dict.SansTopTwentyfive - sansTop25URL = cwe.SansTopTwentyfiveURL - } - if dict, ok := cwe.CweDictJa[cweNum]; ok { - name = dict.Name - url = fmt.Sprintf("http://jvndb.jvn.jp/ja/cwe/%s.html", cweID) - } else { - if dict, ok := cwe.CweDictEn[cweNum]; ok { - name = dict.Name - } - url = fmt.Sprintf("https://cwe.mitre.org/data/definitions/%s.html", cweID) - } - default: - if dict, ok := c[cweNum]; ok && dict.OwaspTopTen2017 != "" { - top10Rank = dict.OwaspTopTen2017 - top10URL = cwe.OwaspTopTen2017GitHubURLEn[dict.OwaspTopTen2017] - } - if dict, ok := c[cweNum]; ok && dict.CweTopTwentyfive2019 != "" { - cweTop25Rank = dict.CweTopTwentyfive2019 - cweTop25URL = cwe.CweTopTwentyfive2019URL - } - if dict, ok := c[cweNum]; ok && dict.SansTopTwentyfive != "" { - sansTop25Rank = dict.SansTopTwentyfive - sansTop25URL = cwe.SansTopTwentyfiveURL - } - url = fmt.Sprintf("https://cwe.mitre.org/data/definitions/%s.html", cweID) - if dict, ok := cwe.CweDictEn[cweNum]; ok { - name = dict.Name - } - } - return +// Container has Container information +type Container struct { + ContainerID string `json:"containerID"` + Name string `json:"name"` + Image string `json:"image"` + Type string `json:"type"` + UUID string `json:"uuid"` } -// CweDictEntry is a entry of CWE -type CweDictEntry struct { - En *cwe.Cwe `json:"en,omitempty"` - Ja *cwe.Cwe `json:"ja,omitempty"` - OwaspTopTen2017 string `json:"owaspTopTen2017"` - CweTopTwentyfive2019 string `json:"cweTopTwentyfive2019"` - SansTopTwentyfive string `json:"sansTopTwentyfive"` +// Platform has platform information +type Platform struct { + Name string `json:"name"` // aws or azure or gcp or other... + InstanceID string `json:"instanceID"` } // Kernel has the Release, version and whether need restart @@ -138,26 +95,7 @@ func (r ScanResult) FilterByCvssOver(over float64) ScanResult { } // FilterIgnoreCves is filter function. -func (r ScanResult) FilterIgnoreCves() ScanResult { - ignoreCves := []string{} - if len(r.Container.Name) == 0 { - //TODO pass by args - ignoreCves = config.Conf.Servers[r.ServerName].IgnoreCves - } else { - //TODO pass by args - if s, ok := config.Conf.Servers[r.ServerName]; ok { - if con, ok := s.Containers[r.Container.Name]; ok { - ignoreCves = con.IgnoreCves - } else { - return r - } - } else { - util.Log.Errorf("%s is not found in config.toml", - r.ServerName) - return r - } - } - +func (r ScanResult) FilterIgnoreCves(ignoreCves []string) ScanResult { filtered := r.ScannedCves.Find(func(v VulnInfo) bool { for _, c := range ignoreCves { if v.CveID == c { @@ -191,25 +129,7 @@ func (r ScanResult) FilterUnfixed(ignoreUnfixed bool) ScanResult { } // FilterIgnorePkgs is filter function. -func (r ScanResult) FilterIgnorePkgs() ScanResult { - var ignorePkgsRegexps []string - if len(r.Container.Name) == 0 { - //TODO pass by args - ignorePkgsRegexps = config.Conf.Servers[r.ServerName].IgnorePkgsRegexp - } else { - if s, ok := config.Conf.Servers[r.ServerName]; ok { - if con, ok := s.Containers[r.Container.Name]; ok { - ignorePkgsRegexps = con.IgnorePkgsRegexp - } else { - return r - } - } else { - util.Log.Errorf("%s is not found in config.toml", - r.ServerName) - return r - } - } - +func (r ScanResult) FilterIgnorePkgs(ignorePkgsRegexps []string) ScanResult { regexps := []*regexp.Regexp{} for _, pkgRegexp := range ignorePkgsRegexps { re, err := regexp.Compile(pkgRegexp) @@ -344,7 +264,7 @@ func (r ScanResult) FormatTextReportHeader() string { buf.WriteString("=") } - pkgs := r.FormatUpdatablePacksSummary() + pkgs := r.FormatUpdatablePkgsSummary() if 0 < len(r.WordPressPackages) { pkgs = fmt.Sprintf("%s, %d WordPress pkgs", pkgs, len(r.WordPressPackages)) } @@ -363,9 +283,10 @@ func (r ScanResult) FormatTextReportHeader() string { pkgs) } -// FormatUpdatablePacksSummary returns a summary of updatable packages -func (r ScanResult) FormatUpdatablePacksSummary() string { - if !r.isDisplayUpdatableNum() { +// FormatUpdatablePkgsSummary returns a summary of updatable packages +func (r ScanResult) FormatUpdatablePkgsSummary() string { + mode := r.Config.Scan.Servers[r.ServerName].Mode + if !r.isDisplayUpdatableNum(mode) { return fmt.Sprintf("%d installed", len(r.Packages)) } @@ -420,16 +341,11 @@ func (r ScanResult) FormatAlertSummary() string { return fmt.Sprintf("en: %d, ja: %d alerts", enCnt, jaCnt) } -func (r ScanResult) isDisplayUpdatableNum() bool { - if r.Family == config.FreeBSD { +func (r ScanResult) isDisplayUpdatableNum(mode config.ScanMode) bool { + if r.Family == constant.FreeBSD { return false } - var mode config.ScanMode - //TODO pass by args - s, _ := config.Conf.Servers[r.ServerName] - mode = s.Mode - if mode.IsOffline() { return false } @@ -438,11 +354,11 @@ func (r ScanResult) isDisplayUpdatableNum() bool { } if mode.IsFast() { switch r.Family { - case config.RedHat, - config.Oracle, - config.Debian, - config.Ubuntu, - config.Raspbian: + case constant.RedHat, + constant.Oracle, + constant.Debian, + constant.Ubuntu, + constant.Raspbian: return false default: return true @@ -456,34 +372,9 @@ func (r ScanResult) IsContainer() bool { return 0 < len(r.Container.ContainerID) } -// IsDeepScanMode checks if the scan mode is deep scan mode. -func (r ScanResult) IsDeepScanMode() bool { - for _, s := range r.Config.Scan.Servers { - if ok := s.Mode.IsDeep(); ok { - return true - } - } - return false -} - -// Container has Container information -type Container struct { - ContainerID string `json:"containerID"` - Name string `json:"name"` - Image string `json:"image"` - Type string `json:"type"` - UUID string `json:"uuid"` -} - -// Platform has platform information -type Platform struct { - Name string `json:"name"` // aws or azure or gcp or other... - InstanceID string `json:"instanceID"` -} - // RemoveRaspbianPackFromResult is for Raspberry Pi and removes the Raspberry Pi dedicated package from ScanResult. func (r ScanResult) RemoveRaspbianPackFromResult() ScanResult { - if r.Family != config.Raspbian { + if r.Family != constant.Raspbian { return r } @@ -528,3 +419,99 @@ func (r ScanResult) ClearFields(targetTagNames []string) ScanResult { } return r } + +// CheckEOL checks the EndOfLife of the OS +func (r *ScanResult) CheckEOL() { + switch r.Family { + case constant.ServerTypePseudo, constant.Raspbian: + return + } + + eol, found := config.GetEOL(r.Family, r.Release) + if !found { + r.Warnings = append(r.Warnings, + fmt.Sprintf("Failed to check EOL. Register the issue to https://github.com/future-architect/vuls/issues with the information in `Family: %s Release: %s`", + r.Family, r.Release)) + return + } + + now := time.Now() + if eol.IsStandardSupportEnded(now) { + r.Warnings = append(r.Warnings, "Standard OS support is EOL(End-of-Life). Purchase extended support if available or Upgrading your OS is strongly recommended.") + if eol.ExtendedSupportUntil.IsZero() { + return + } + if !eol.IsExtendedSuppportEnded(now) { + r.Warnings = append(r.Warnings, + fmt.Sprintf("Extended support available until %s. Check the vendor site.", + eol.ExtendedSupportUntil.Format("2006-01-02"))) + } else { + r.Warnings = append(r.Warnings, + "Extended support is also EOL. There are many Vulnerabilities that are not detected, Upgrading your OS strongly recommended.") + } + } else if !eol.StandardSupportUntil.IsZero() && + now.AddDate(0, 3, 0).After(eol.StandardSupportUntil) { + r.Warnings = append(r.Warnings, + fmt.Sprintf("Standard OS support will be end in 3 months. EOL date: %s", + eol.StandardSupportUntil.Format("2006-01-02"))) + } +} + +// CweDict is a dictionary for CWE +type CweDict map[string]CweDictEntry + +// Get the name, url, top10URL for the specified cweID, lang +func (c CweDict) Get(cweID, lang string) (name, url, top10Rank, top10URL, cweTop25Rank, cweTop25URL, sansTop25Rank, sansTop25URL string) { + cweNum := strings.TrimPrefix(cweID, "CWE-") + switch lang { + case "ja": + if dict, ok := c[cweNum]; ok && dict.OwaspTopTen2017 != "" { + top10Rank = dict.OwaspTopTen2017 + top10URL = cwe.OwaspTopTen2017GitHubURLJa[dict.OwaspTopTen2017] + } + if dict, ok := c[cweNum]; ok && dict.CweTopTwentyfive2019 != "" { + cweTop25Rank = dict.CweTopTwentyfive2019 + cweTop25URL = cwe.CweTopTwentyfive2019URL + } + if dict, ok := c[cweNum]; ok && dict.SansTopTwentyfive != "" { + sansTop25Rank = dict.SansTopTwentyfive + sansTop25URL = cwe.SansTopTwentyfiveURL + } + if dict, ok := cwe.CweDictJa[cweNum]; ok { + name = dict.Name + url = fmt.Sprintf("http://jvndb.jvn.jp/ja/cwe/%s.html", cweID) + } else { + if dict, ok := cwe.CweDictEn[cweNum]; ok { + name = dict.Name + } + url = fmt.Sprintf("https://cwe.mitre.org/data/definitions/%s.html", cweID) + } + default: + if dict, ok := c[cweNum]; ok && dict.OwaspTopTen2017 != "" { + top10Rank = dict.OwaspTopTen2017 + top10URL = cwe.OwaspTopTen2017GitHubURLEn[dict.OwaspTopTen2017] + } + if dict, ok := c[cweNum]; ok && dict.CweTopTwentyfive2019 != "" { + cweTop25Rank = dict.CweTopTwentyfive2019 + cweTop25URL = cwe.CweTopTwentyfive2019URL + } + if dict, ok := c[cweNum]; ok && dict.SansTopTwentyfive != "" { + sansTop25Rank = dict.SansTopTwentyfive + sansTop25URL = cwe.SansTopTwentyfiveURL + } + url = fmt.Sprintf("https://cwe.mitre.org/data/definitions/%s.html", cweID) + if dict, ok := cwe.CweDictEn[cweNum]; ok { + name = dict.Name + } + } + return +} + +// CweDictEntry is a entry of CWE +type CweDictEntry struct { + En *cwe.Cwe `json:"en,omitempty"` + Ja *cwe.Cwe `json:"ja,omitempty"` + OwaspTopTen2017 string `json:"owaspTopTen2017"` + CweTopTwentyfive2019 string `json:"cweTopTwentyfive2019"` + SansTopTwentyfive string `json:"sansTopTwentyfive"` +} diff --git a/models/scanresults_test.go b/models/scanresults_test.go index 64966283..eeb42e69 100644 --- a/models/scanresults_test.go +++ b/models/scanresults_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/k0kubun/pp" ) @@ -233,80 +234,10 @@ func TestFilterIgnoreCveIDs(t *testing.T) { }, } for _, tt := range tests { - config.Conf.Servers = map[string]config.ServerInfo{ - "name": {IgnoreCves: tt.in.cves}, - } - actual := tt.in.rs.FilterIgnoreCves() - for k := range tt.out.ScannedCves { - if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) { - o := pp.Sprintf("%v", tt.out.ScannedCves[k]) - a := pp.Sprintf("%v", actual.ScannedCves[k]) - t.Errorf("[%s] expected: %v\n actual: %v\n", k, o, a) - } - } - for k := range actual.ScannedCves { - if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) { - o := pp.Sprintf("%v", tt.out.ScannedCves[k]) - a := pp.Sprintf("%v", actual.ScannedCves[k]) - t.Errorf("[%s] expected: %v\n actual: %v\n", k, o, a) - } - } - } -} - -func TestFilterIgnoreCveIDsContainer(t *testing.T) { - type in struct { - cves []string - rs ScanResult - } - var tests = []struct { - in in - out ScanResult - }{ - { - in: in{ - cves: []string{"CVE-2017-0002"}, - rs: ScanResult{ - ServerName: "name", - Container: Container{Name: "dockerA"}, - ScannedCves: VulnInfos{ - "CVE-2017-0001": { - CveID: "CVE-2017-0001", - }, - "CVE-2017-0002": { - CveID: "CVE-2017-0002", - }, - "CVE-2017-0003": { - CveID: "CVE-2017-0003", - }, - }, - }, - }, - out: ScanResult{ - ServerName: "name", - Container: Container{Name: "dockerA"}, - ScannedCves: VulnInfos{ - "CVE-2017-0001": { - CveID: "CVE-2017-0001", - }, - "CVE-2017-0003": { - CveID: "CVE-2017-0003", - }, - }, - }, - }, - } - for _, tt := range tests { - config.Conf.Servers = map[string]config.ServerInfo{ - "name": { - Containers: map[string]config.ContainerSetting{ - "dockerA": { - IgnoreCves: tt.in.cves, - }, - }, - }, - } - actual := tt.in.rs.FilterIgnoreCves() + // config.Conf.Servers = map[string]config.ServerInfo{ + // "name": {IgnoreCves: tt.in.cves}, + // } + actual := tt.in.rs.FilterIgnoreCves(tt.in.cves) for k := range tt.out.ScannedCves { if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) { o := pp.Sprintf("%v", tt.out.ScannedCves[k]) @@ -491,131 +422,7 @@ func TestFilterIgnorePkgs(t *testing.T) { }, } for _, tt := range tests { - config.Conf.Servers = map[string]config.ServerInfo{ - "name": {IgnorePkgsRegexp: tt.in.ignorePkgsRegexp}, - } - actual := tt.in.rs.FilterIgnorePkgs() - for k := range tt.out.ScannedCves { - if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) { - o := pp.Sprintf("%v", tt.out.ScannedCves[k]) - a := pp.Sprintf("%v", actual.ScannedCves[k]) - t.Errorf("[%s] expected: %v\n actual: %v\n", k, o, a) - } - } - for k := range actual.ScannedCves { - if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) { - o := pp.Sprintf("%v", tt.out.ScannedCves[k]) - a := pp.Sprintf("%v", actual.ScannedCves[k]) - t.Errorf("[%s] expected: %v\n actual: %v\n", k, o, a) - } - } - } -} - -func TestFilterIgnorePkgsContainer(t *testing.T) { - type in struct { - ignorePkgsRegexp []string - rs ScanResult - } - var tests = []struct { - in in - out ScanResult - }{ - { - in: in{ - ignorePkgsRegexp: []string{"^kernel"}, - rs: ScanResult{ - ServerName: "name", - Container: Container{Name: "dockerA"}, - ScannedCves: VulnInfos{ - "CVE-2017-0001": { - CveID: "CVE-2017-0001", - AffectedPackages: PackageFixStatuses{ - {Name: "kernel"}, - }, - }, - "CVE-2017-0002": { - CveID: "CVE-2017-0002", - }, - }, - }, - }, - out: ScanResult{ - ServerName: "name", - Container: Container{Name: "dockerA"}, - ScannedCves: VulnInfos{ - "CVE-2017-0002": { - CveID: "CVE-2017-0002", - }, - }, - }, - }, - { - in: in{ - ignorePkgsRegexp: []string{"^kernel"}, - rs: ScanResult{ - ServerName: "name", - Container: Container{Name: "dockerA"}, - ScannedCves: VulnInfos{ - "CVE-2017-0001": { - CveID: "CVE-2017-0001", - AffectedPackages: PackageFixStatuses{ - {Name: "kernel"}, - {Name: "vim"}, - }, - }, - }, - }, - }, - out: ScanResult{ - ServerName: "name", - Container: Container{Name: "dockerA"}, - ScannedCves: VulnInfos{ - "CVE-2017-0001": { - CveID: "CVE-2017-0001", - AffectedPackages: PackageFixStatuses{ - {Name: "kernel"}, - {Name: "vim"}, - }, - }, - }, - }, - }, - { - in: in{ - ignorePkgsRegexp: []string{"^kernel", "^vim", "^bind"}, - rs: ScanResult{ - ServerName: "name", - Container: Container{Name: "dockerA"}, - ScannedCves: VulnInfos{ - "CVE-2017-0001": { - CveID: "CVE-2017-0001", - AffectedPackages: PackageFixStatuses{ - {Name: "kernel"}, - {Name: "vim"}, - }, - }, - }, - }, - }, - out: ScanResult{ - ServerName: "name", - Container: Container{Name: "dockerA"}, - ScannedCves: VulnInfos{}, - }, - }, - } - for _, tt := range tests { - config.Conf.Servers = map[string]config.ServerInfo{ - "name": { - Containers: map[string]config.ContainerSetting{ - "dockerA": { - IgnorePkgsRegexp: tt.in.ignorePkgsRegexp, - }, - }, - }, - } - actual := tt.in.rs.FilterIgnorePkgs() + actual := tt.in.rs.FilterIgnorePkgs(tt.in.ignorePkgsRegexp) for k := range tt.out.ScannedCves { if !reflect.DeepEqual(tt.out.ScannedCves[k], actual.ScannedCves[k]) { o := pp.Sprintf("%v", tt.out.ScannedCves[k]) @@ -653,52 +460,52 @@ func TestIsDisplayUpdatableNum(t *testing.T) { }, { mode: []byte{config.Fast}, - family: config.RedHat, + family: constant.RedHat, expected: false, }, { mode: []byte{config.Fast}, - family: config.Oracle, + family: constant.Oracle, expected: false, }, { mode: []byte{config.Fast}, - family: config.Debian, + family: constant.Debian, expected: false, }, { mode: []byte{config.Fast}, - family: config.Ubuntu, + family: constant.Ubuntu, expected: false, }, { mode: []byte{config.Fast}, - family: config.Raspbian, + family: constant.Raspbian, expected: false, }, { mode: []byte{config.Fast}, - family: config.CentOS, + family: constant.CentOS, expected: true, }, { mode: []byte{config.Fast}, - family: config.Amazon, + family: constant.Amazon, expected: true, }, { mode: []byte{config.Fast}, - family: config.FreeBSD, + family: constant.FreeBSD, expected: false, }, { mode: []byte{config.Fast}, - family: config.OpenSUSE, + family: constant.OpenSUSE, expected: true, }, { mode: []byte{config.Fast}, - family: config.Alpine, + family: constant.Alpine, expected: true, }, } @@ -708,14 +515,11 @@ func TestIsDisplayUpdatableNum(t *testing.T) { for _, m := range tt.mode { mode.Set(m) } - config.Conf.Servers = map[string]config.ServerInfo{ - "name": {Mode: mode}, - } r := ScanResult{ ServerName: "name", Family: tt.family, } - act := r.isDisplayUpdatableNum() + act := r.isDisplayUpdatableNum(mode) if tt.expected != act { t.Errorf("[%d] expected %#v, actual %#v", i, tt.expected, act) } diff --git a/models/vulninfos.go b/models/vulninfos.go index 82fed774..8ea05679 100644 --- a/models/vulninfos.go +++ b/models/vulninfos.go @@ -7,7 +7,6 @@ import ( "strings" "time" - "github.com/future-architect/vuls/config" exploitmodels "github.com/vulsio/go-exploitdb/models" ) @@ -78,19 +77,14 @@ func (v VulnInfos) CountGroupBySeverity() map[string]int { } // FormatCveSummary summarize the number of CVEs group by CVSSv2 Severity -func (v VulnInfos) FormatCveSummary() (line string) { +func (v VulnInfos) FormatCveSummary() string { m := v.CountGroupBySeverity() - if config.Conf.IgnoreUnscoredCves { - line = fmt.Sprintf("Total: %d (Critical:%d High:%d Medium:%d Low:%d)", - m["High"]+m["Medium"]+m["Low"], m["Critical"], m["High"], m["Medium"], m["Low"]) - } else { - line = fmt.Sprintf("Total: %d (Critical:%d High:%d Medium:%d Low:%d ?:%d)", - m["High"]+m["Medium"]+m["Low"]+m["Unknown"], - m["Critical"], m["High"], m["Medium"], m["Low"], m["Unknown"]) - } + line := fmt.Sprintf("Total: %d (Critical:%d High:%d Medium:%d Low:%d ?:%d)", + m["High"]+m["Medium"]+m["Low"]+m["Unknown"], + m["Critical"], m["High"], m["Medium"], m["Low"], m["Unknown"]) - if config.Conf.DiffMinus || config.Conf.DiffPlus { - nPlus, nMinus := v.CountDiff() + nPlus, nMinus := v.CountDiff() + if 0 < nPlus || 0 < nMinus { line = fmt.Sprintf("%s +%d -%d", line, nPlus, nMinus) } return line @@ -266,8 +260,8 @@ const ( ) // CveIDDiffFormat format CVE-ID for diff mode -func (v VulnInfo) CveIDDiffFormat(isDiffMode bool) string { - if isDiffMode { +func (v VulnInfo) CveIDDiffFormat() string { + if v.DiffStatus != "" { return fmt.Sprintf("%s %s", v.DiffStatus, v.CveID) } return fmt.Sprintf("%s", v.CveID) diff --git a/oval/alpine.go b/oval/alpine.go index 3e7cd21d..e2e3001c 100644 --- a/oval/alpine.go +++ b/oval/alpine.go @@ -4,6 +4,7 @@ package oval import ( "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" "github.com/kotakanbe/goval-dictionary/db" @@ -18,7 +19,7 @@ type Alpine struct { func NewAlpine() Alpine { return Alpine{ Base{ - family: config.Alpine, + family: constant.Alpine, }, } } diff --git a/oval/debian.go b/oval/debian.go index 3cd7616e..fe7b4557 100644 --- a/oval/debian.go +++ b/oval/debian.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" "github.com/kotakanbe/goval-dictionary/db" @@ -40,7 +41,7 @@ func (o DebianBase) update(r *models.ScanResult, defPacks defPacks) { defPacks.def.Debian.CveID) cveContents = models.CveContents{} } - if r.Family != config.Raspbian { + if r.Family != constant.Raspbian { vinfo.Confidences.AppendIfMissing(models.OvalMatch) } else { if len(vinfo.Confidences) == 0 { @@ -113,7 +114,7 @@ func NewDebian() Debian { return Debian{ DebianBase{ Base{ - family: config.Debian, + family: constant.Debian, }, }, } @@ -140,7 +141,7 @@ func (o Debian) FillWithOval(driver db.DB, r *models.ScanResult) (nCVEs int, err var relatedDefs ovalResult if config.Conf.OvalDict.IsFetchViaHTTP() { - if r.Family != config.Raspbian { + if r.Family != constant.Raspbian { if relatedDefs, err = getDefsByPackNameViaHTTP(r); err != nil { return 0, err } @@ -152,7 +153,7 @@ func (o Debian) FillWithOval(driver db.DB, r *models.ScanResult) (nCVEs int, err } } } else { - if r.Family != config.Raspbian { + if r.Family != constant.Raspbian { if relatedDefs, err = getDefsByPackNameFromOvalDB(driver, r); err != nil { return 0, err } @@ -203,7 +204,7 @@ func NewUbuntu() Ubuntu { return Ubuntu{ DebianBase{ Base{ - family: config.Ubuntu, + family: constant.Ubuntu, }, }, } diff --git a/oval/redhat.go b/oval/redhat.go index b358aec5..a034099b 100644 --- a/oval/redhat.go +++ b/oval/redhat.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" "github.com/kotakanbe/goval-dictionary/db" @@ -143,7 +144,7 @@ func (o RedHatBase) update(r *models.ScanResult, defPacks defPacks) (nCVEs int) func (o RedHatBase) convertToDistroAdvisory(def *ovalmodels.Definition) *models.DistroAdvisory { advisoryID := def.Title - if (o.family == config.RedHat || o.family == config.CentOS) && len(advisoryID) > 0 { + if (o.family == constant.RedHat || o.family == constant.CentOS) && len(advisoryID) > 0 { ss := strings.Fields(def.Title) advisoryID = strings.TrimSuffix(ss[0], ":") } @@ -250,7 +251,7 @@ func NewRedhat() RedHat { return RedHat{ RedHatBase{ Base{ - family: config.RedHat, + family: constant.RedHat, }, }, } @@ -266,7 +267,7 @@ func NewCentOS() CentOS { return CentOS{ RedHatBase{ Base{ - family: config.CentOS, + family: constant.CentOS, }, }, } @@ -282,7 +283,7 @@ func NewOracle() Oracle { return Oracle{ RedHatBase{ Base{ - family: config.Oracle, + family: constant.Oracle, }, }, } @@ -299,7 +300,7 @@ func NewAmazon() Amazon { return Amazon{ RedHatBase{ Base{ - family: config.Amazon, + family: constant.Amazon, }, }, } diff --git a/oval/suse.go b/oval/suse.go index 9de65619..a5a138d6 100644 --- a/oval/suse.go +++ b/oval/suse.go @@ -4,6 +4,7 @@ package oval import ( "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" "github.com/kotakanbe/goval-dictionary/db" @@ -20,7 +21,7 @@ func NewSUSE() SUSE { // TODO implement other family return SUSE{ Base{ - family: config.SUSEEnterpriseServer, + family: constant.SUSEEnterpriseServer, }, } } diff --git a/oval/util.go b/oval/util.go index b43d1c6e..35fcf5aa 100644 --- a/oval/util.go +++ b/oval/util.go @@ -10,6 +10,7 @@ import ( "github.com/cenkalti/backoff" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" apkver "github.com/knqyf263/go-apk-version" @@ -300,7 +301,7 @@ func isOvalDefAffected(def ovalmodels.Definition, req request, family string, ru if running.Release != "" { switch family { - case config.RedHat, config.CentOS: + case constant.RedHat, constant.CentOS: // For kernel related packages, ignore OVAL information with different major versions if _, ok := kernelRelatedPackNames[ovalPack.Name]; ok { if util.Major(ovalPack.Version) != util.Major(running.Release) { @@ -329,12 +330,12 @@ func isOvalDefAffected(def ovalmodels.Definition, req request, family string, ru // If the version of installed is less than in OVAL switch family { - case config.RedHat, - config.Amazon, - config.SUSEEnterpriseServer, - config.Debian, - config.Ubuntu, - config.Raspbian: + case constant.RedHat, + constant.Amazon, + constant.SUSEEnterpriseServer, + constant.Debian, + constant.Ubuntu, + constant.Raspbian: // Use fixed state in OVAL for these distros. return true, false, ovalPack.Version } @@ -365,9 +366,9 @@ func isOvalDefAffected(def ovalmodels.Definition, req request, family string, ru func lessThan(family, newVer string, packInOVAL ovalmodels.Package) (bool, error) { switch family { - case config.Debian, - config.Ubuntu, - config.Raspbian: + case constant.Debian, + constant.Ubuntu, + constant.Raspbian: vera, err := debver.NewVersion(newVer) if err != nil { return false, err @@ -378,7 +379,7 @@ func lessThan(family, newVer string, packInOVAL ovalmodels.Package) (bool, error } return vera.LessThan(verb), nil - case config.Alpine: + case constant.Alpine: vera, err := apkver.NewVersion(newVer) if err != nil { return false, err @@ -389,15 +390,15 @@ func lessThan(family, newVer string, packInOVAL ovalmodels.Package) (bool, error } return vera.LessThan(verb), nil - case config.Oracle, - config.SUSEEnterpriseServer, - config.Amazon: + case constant.Oracle, + constant.SUSEEnterpriseServer, + constant.Amazon: vera := rpmver.NewVersion(newVer) verb := rpmver.NewVersion(packInOVAL.Version) return vera.LessThan(verb), nil - case config.RedHat, - config.CentOS: + case constant.RedHat, + constant.CentOS: vera := rpmver.NewVersion(centOSVersionToRHEL(newVer)) verb := rpmver.NewVersion(centOSVersionToRHEL(packInOVAL.Version)) return vera.LessThan(verb), nil diff --git a/oval/util_test.go b/oval/util_test.go index cefb8c67..c5f1c5d0 100644 --- a/oval/util_test.go +++ b/oval/util_test.go @@ -7,7 +7,7 @@ import ( "sort" "testing" - "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" ovalmodels "github.com/kotakanbe/goval-dictionary/models" ) @@ -1030,7 +1030,7 @@ func TestIsOvalDefAffected(t *testing.T) { // For kernel related packages, ignore OVAL with different major versions { in: in{ - family: config.CentOS, + family: constant.CentOS, def: ovalmodels.Definition{ AffectedPacks: []ovalmodels.Package{ { @@ -1054,7 +1054,7 @@ func TestIsOvalDefAffected(t *testing.T) { }, { in: in{ - family: config.CentOS, + family: constant.CentOS, def: ovalmodels.Definition{ AffectedPacks: []ovalmodels.Package{ { @@ -1080,7 +1080,7 @@ func TestIsOvalDefAffected(t *testing.T) { // dnf module { in: in{ - family: config.RedHat, + family: constant.RedHat, def: ovalmodels.Definition{ AffectedPacks: []ovalmodels.Package{ { @@ -1106,7 +1106,7 @@ func TestIsOvalDefAffected(t *testing.T) { // dnf module 2 { in: in{ - family: config.RedHat, + family: constant.RedHat, def: ovalmodels.Definition{ AffectedPacks: []ovalmodels.Package{ { @@ -1131,7 +1131,7 @@ func TestIsOvalDefAffected(t *testing.T) { // dnf module 3 { in: in{ - family: config.RedHat, + family: constant.RedHat, def: ovalmodels.Definition{ AffectedPacks: []ovalmodels.Package{ { diff --git a/report/azureblob.go b/reporter/azureblob.go similarity index 80% rename from report/azureblob.go rename to reporter/azureblob.go index f2b9d4e8..f4624f55 100644 --- a/report/azureblob.go +++ b/reporter/azureblob.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "bytes" @@ -14,7 +14,13 @@ import ( ) // AzureBlobWriter writes results to AzureBlob -type AzureBlobWriter struct{} +type AzureBlobWriter struct { + FormatJSON bool + FormatFullText bool + FormatOneLineText bool + FormatList bool + Gzip bool +} // Write results to Azure Blob storage func (w AzureBlobWriter) Write(rs ...models.ScanResult) (err error) { @@ -27,41 +33,41 @@ func (w AzureBlobWriter) Write(rs ...models.ScanResult) (err error) { return err } - if c.Conf.FormatOneLineText { + if w.FormatOneLineText { timestr := rs[0].ScannedAt.Format(time.RFC3339) k := fmt.Sprintf(timestr + "/summary.txt") text := formatOneLineSummary(rs...) b := []byte(text) - if err := createBlockBlob(cli, k, b); err != nil { + if err := createBlockBlob(cli, k, b, w.Gzip); err != nil { return err } } for _, r := range rs { key := r.ReportKeyName() - if c.Conf.FormatJSON { + if w.FormatJSON { k := key + ".json" var b []byte if b, err = json.Marshal(r); err != nil { return xerrors.Errorf("Failed to Marshal to JSON: %w", err) } - if err := createBlockBlob(cli, k, b); err != nil { + if err := createBlockBlob(cli, k, b, w.Gzip); err != nil { return err } } - if c.Conf.FormatList { + if w.FormatList { k := key + "_short.txt" b := []byte(formatList(r)) - if err := createBlockBlob(cli, k, b); err != nil { + if err := createBlockBlob(cli, k, b, w.Gzip); err != nil { return err } } - if c.Conf.FormatFullText { + if w.FormatFullText { k := key + "_full.txt" b := []byte(formatFullPlainText(r)) - if err := createBlockBlob(cli, k, b); err != nil { + if err := createBlockBlob(cli, k, b, w.Gzip); err != nil { return err } } @@ -101,9 +107,9 @@ func getBlobClient() (storage.BlobStorageClient, error) { return api.GetBlobService(), nil } -func createBlockBlob(cli storage.BlobStorageClient, k string, b []byte) error { +func createBlockBlob(cli storage.BlobStorageClient, k string, b []byte, gzip bool) error { var err error - if c.Conf.GZIP { + if gzip { if b, err = gz(b); err != nil { return err } diff --git a/report/chatwork.go b/reporter/chatwork.go similarity index 96% rename from report/chatwork.go rename to reporter/chatwork.go index 2dd8f075..4b06c46b 100644 --- a/report/chatwork.go +++ b/reporter/chatwork.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "context" @@ -38,7 +38,7 @@ func (w ChatWorkWriter) Write(rs ...models.ScanResult) (err error) { vinfo.CveID, strconv.FormatFloat(maxCvss.Value.Score, 'f', 1, 64), severity, - vinfo.Summaries(config.Conf.Lang, r.Family)[0].Value) + vinfo.Summaries(r.Lang, r.Family)[0].Value) if err = chatWorkpostMessage(conf.Room, conf.APIToken, message); err != nil { return err diff --git a/report/email.go b/reporter/email.go similarity index 90% rename from report/email.go rename to reporter/email.go index 79717c0b..cd626488 100644 --- a/report/email.go +++ b/reporter/email.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "crypto/tls" @@ -16,7 +16,11 @@ import ( ) // EMailWriter send mail -type EMailWriter struct{} +type EMailWriter struct { + FormatOneEMail bool + FormatOneLineText bool + FormatList bool +} func (w EMailWriter) Write(rs ...models.ScanResult) (err error) { conf := config.Conf @@ -24,7 +28,7 @@ func (w EMailWriter) Write(rs ...models.ScanResult) (err error) { sender := NewEMailSender() m := map[string]int{} for _, r := range rs { - if conf.FormatOneEMail { + if w.FormatOneEMail { message += formatFullPlainText(r) + "\r\n\r\n" mm := r.ScannedCves.CountGroupBySeverity() keys := []string{"High", "Medium", "Low", "Unknown"} @@ -42,12 +46,12 @@ func (w EMailWriter) Write(rs ...models.ScanResult) (err error) { r.ServerInfo(), r.ScannedCves.FormatCveSummary()) } - if conf.FormatList { + if w.FormatList { message = formatList(r) } else { message = formatFullPlainText(r) } - if conf.FormatOneLineText { + if w.FormatOneLineText { message = fmt.Sprintf("One Line Summary\r\n================\r\n%s", formatOneLineSummary(r)) } if err := sender.Send(subject, message); err != nil { @@ -55,19 +59,15 @@ func (w EMailWriter) Write(rs ...models.ScanResult) (err error) { } } } - var summary string - if config.Conf.IgnoreUnscoredCves { - summary = fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d)", - m["High"]+m["Medium"]+m["Low"], m["High"], m["Medium"], m["Low"]) - } else { - summary = fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d ?:%d)", - m["High"]+m["Medium"]+m["Low"]+m["Unknown"], - m["High"], m["Medium"], m["Low"], m["Unknown"]) - } + + summary := fmt.Sprintf("Total: %d (High:%d Medium:%d Low:%d ?:%d)", + m["High"]+m["Medium"]+m["Low"]+m["Unknown"], + m["High"], m["Medium"], m["Low"], m["Unknown"]) + origmessage := message - if conf.FormatOneEMail { + if w.FormatOneEMail { message = fmt.Sprintf("One Line Summary\r\n================\r\n%s", formatOneLineSummary(rs...)) - if !conf.FormatOneLineText { + if !w.FormatOneLineText { message += fmt.Sprintf("\r\n\r\n%s", origmessage) } diff --git a/report/http.go b/reporter/http.go similarity index 98% rename from report/http.go rename to reporter/http.go index 3a8774ca..912b1848 100644 --- a/report/http.go +++ b/reporter/http.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "bytes" diff --git a/report/localfile.go b/reporter/localfile.go similarity index 66% rename from report/localfile.go rename to reporter/localfile.go index 33d417bc..f4bfd624 100644 --- a/report/localfile.go +++ b/reporter/localfile.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "encoding/json" @@ -6,21 +6,28 @@ import ( "os" "path/filepath" - c "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" "golang.org/x/xerrors" ) // LocalFileWriter writes results to a local file. type LocalFileWriter struct { - CurrentDir string + CurrentDir string + DiffPlus bool + DiffMinus bool + FormatJSON bool + FormatCsv bool + FormatFullText bool + FormatOneLineText bool + FormatList bool + Gzip bool } func (w LocalFileWriter) Write(rs ...models.ScanResult) (err error) { - if c.Conf.FormatOneLineText { + if w.FormatOneLineText { path := filepath.Join(w.CurrentDir, "summary.txt") text := formatOneLineSummary(rs...) - if err := writeFile(path, []byte(text), 0600); err != nil { + if err := w.writeFile(path, []byte(text), 0600); err != nil { return xerrors.Errorf( "Failed to write to file. path: %s, err: %w", path, err) @@ -30,48 +37,48 @@ func (w LocalFileWriter) Write(rs ...models.ScanResult) (err error) { for _, r := range rs { path := filepath.Join(w.CurrentDir, r.ReportFileName()) - if c.Conf.FormatJSON { + if w.FormatJSON { p := path + ".json" - if c.Conf.DiffPlus || c.Conf.DiffMinus { + if w.DiffPlus || w.DiffMinus { p = path + "_diff.json" } var b []byte if b, err = json.MarshalIndent(r, "", " "); err != nil { return xerrors.Errorf("Failed to Marshal to JSON: %w", err) } - if err := writeFile(p, b, 0600); err != nil { + if err := w.writeFile(p, b, 0600); err != nil { return xerrors.Errorf("Failed to write JSON. path: %s, err: %w", p, err) } } - if c.Conf.FormatList { + if w.FormatList { p := path + "_short.txt" - if c.Conf.DiffPlus || c.Conf.DiffMinus { + if w.DiffPlus || w.DiffMinus { p = path + "_short_diff.txt" } - if err := writeFile( + if err := w.writeFile( p, []byte(formatList(r)), 0600); err != nil { return xerrors.Errorf( "Failed to write text files. path: %s, err: %w", p, err) } } - if c.Conf.FormatFullText { + if w.FormatFullText { p := path + "_full.txt" - if c.Conf.DiffPlus || c.Conf.DiffMinus { + if w.DiffPlus || w.DiffMinus { p = path + "_full_diff.txt" } - if err := writeFile( + if err := w.writeFile( p, []byte(formatFullPlainText(r)), 0600); err != nil { return xerrors.Errorf( "Failed to write text files. path: %s, err: %w", p, err) } } - if c.Conf.FormatCsvList { + if w.FormatCsv { p := path + ".csv" - if c.Conf.DiffPlus || c.Conf.DiffMinus { + if w.DiffPlus || w.DiffMinus { p = path + "_diff.csv" } if err := formatCsvList(r, p); err != nil { @@ -83,10 +90,10 @@ func (w LocalFileWriter) Write(rs ...models.ScanResult) (err error) { return nil } -func writeFile(path string, data []byte, perm os.FileMode) error { - var err error - if c.Conf.GZIP { - if data, err = gz(data); err != nil { +func (w LocalFileWriter) writeFile(path string, data []byte, perm os.FileMode) (err error) { + if w.Gzip { + data, err = gz(data) + if err != nil { return err } path += ".gz" diff --git a/report/s3.go b/reporter/s3.go similarity index 84% rename from report/s3.go rename to reporter/s3.go index c87b87b5..5f9f60b6 100644 --- a/report/s3.go +++ b/reporter/s3.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "bytes" @@ -20,7 +20,13 @@ import ( ) // S3Writer writes results to S3 -type S3Writer struct{} +type S3Writer struct { + FormatJSON bool + FormatFullText bool + FormatOneLineText bool + FormatList bool + Gzip bool +} func getS3() (*s3.S3, error) { ses, err := session.NewSession() @@ -54,40 +60,40 @@ func (w S3Writer) Write(rs ...models.ScanResult) (err error) { return err } - if c.Conf.FormatOneLineText { + if w.FormatOneLineText { timestr := rs[0].ScannedAt.Format(time.RFC3339) k := fmt.Sprintf(timestr + "/summary.txt") text := formatOneLineSummary(rs...) - if err := putObject(svc, k, []byte(text)); err != nil { + if err := putObject(svc, k, []byte(text), w.Gzip); err != nil { return err } } for _, r := range rs { key := r.ReportKeyName() - if c.Conf.FormatJSON { + if w.FormatJSON { k := key + ".json" var b []byte if b, err = json.Marshal(r); err != nil { return xerrors.Errorf("Failed to Marshal to JSON: %w", err) } - if err := putObject(svc, k, b); err != nil { + if err := putObject(svc, k, b, w.Gzip); err != nil { return err } } - if c.Conf.FormatList { + if w.FormatList { k := key + "_short.txt" text := formatList(r) - if err := putObject(svc, k, []byte(text)); err != nil { + if err := putObject(svc, k, []byte(text), w.Gzip); err != nil { return err } } - if c.Conf.FormatFullText { + if w.FormatFullText { k := key + "_full.txt" text := formatFullPlainText(r) - if err := putObject(svc, k, []byte(text)); err != nil { + if err := putObject(svc, k, []byte(text), w.Gzip); err != nil { return err } } @@ -124,9 +130,9 @@ func CheckIfBucketExists() error { return nil } -func putObject(svc *s3.S3, k string, b []byte) error { +func putObject(svc *s3.S3, k string, b []byte, gzip bool) error { var err error - if c.Conf.GZIP { + if gzip { if b, err = gz(b); err != nil { return err } diff --git a/report/slack.go b/reporter/slack.go similarity index 87% rename from report/slack.go rename to reporter/slack.go index 061f3eac..230d11ca 100644 --- a/report/slack.go +++ b/reporter/slack.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "encoding/json" @@ -16,6 +16,13 @@ import ( "golang.org/x/xerrors" ) +// SlackWriter send report to slack +type SlackWriter struct { + FormatOneLineText bool + lang string + osFamily string +} + type message struct { Text string `json:"text"` Username string `json:"username"` @@ -24,15 +31,13 @@ type message struct { Attachments []slack.Attachment `json:"attachments"` } -// SlackWriter send report to slack -type SlackWriter struct{} - func (w SlackWriter) Write(rs ...models.ScanResult) (err error) { conf := config.Conf.Slack channel := conf.Channel token := conf.LegacyToken for _, r := range rs { + w.lang, w.osFamily = r.Lang, r.Family if channel == "${servername}" { channel = fmt.Sprintf("#%s", r.ServerName) } @@ -42,7 +47,7 @@ func (w SlackWriter) Write(rs ...models.ScanResult) (err error) { // https://api.slack.com/methods/chat.postMessage maxAttachments := 100 m := map[int][]slack.Attachment{} - for i, a := range toSlackAttachments(r) { + for i, a := range w.toSlackAttachments(r) { m[i/maxAttachments] = append(m[i/maxAttachments], a) } chunkKeys := []int{} @@ -52,7 +57,7 @@ func (w SlackWriter) Write(rs ...models.ScanResult) (err error) { sort.Ints(chunkKeys) summary := fmt.Sprintf("%s\n%s", - getNotifyUsers(config.Conf.Slack.NotifyUsers), + w.getNotifyUsers(config.Conf.Slack.NotifyUsers), formatOneLineSummary(r)) // Send slack by API @@ -72,7 +77,7 @@ func (w SlackWriter) Write(rs ...models.ScanResult) (err error) { return err } - if config.Conf.FormatOneLineText || 0 < len(r.Errors) { + if w.FormatOneLineText || 0 < len(r.Errors) { continue } @@ -98,11 +103,11 @@ func (w SlackWriter) Write(rs ...models.ScanResult) (err error) { IconEmoji: conf.IconEmoji, Channel: channel, } - if err := send(msg); err != nil { + if err := w.send(msg); err != nil { return err } - if config.Conf.FormatOneLineText || 0 < len(r.Errors) { + if w.FormatOneLineText || 0 < len(r.Errors) { continue } @@ -119,7 +124,7 @@ func (w SlackWriter) Write(rs ...models.ScanResult) (err error) { Channel: channel, Attachments: m[k], } - if err = send(msg); err != nil { + if err = w.send(msg); err != nil { return err } } @@ -128,7 +133,7 @@ func (w SlackWriter) Write(rs ...models.ScanResult) (err error) { return nil } -func send(msg message) error { +func (w SlackWriter) send(msg message) error { conf := config.Conf.Slack count, retryMax := 0, 10 @@ -162,7 +167,7 @@ func send(msg message) error { return nil } -func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) { +func (w SlackWriter) toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) { vinfos := r.ScannedCves.ToSortedSlice() for _, vinfo := range vinfos { @@ -206,9 +211,9 @@ func toSlackAttachments(r models.ScanResult) (attaches []slack.Attachment) { } a := slack.Attachment{ - Title: vinfo.CveIDDiffFormat(config.Conf.DiffMinus || config.Conf.DiffPlus), + Title: vinfo.CveIDDiffFormat(), TitleLink: "https://nvd.nist.gov/vuln/detail/" + vinfo.CveID, - Text: attachmentText(vinfo, r.Family, r.CweDict, r.Packages), + Text: w.attachmentText(vinfo, r.CweDict, r.Packages), MarkdownIn: []string{"text", "pretext"}, Fields: []slack.AttachmentField{ { @@ -244,7 +249,7 @@ func cvssColor(cvssScore float64) string { } } -func attachmentText(vinfo models.VulnInfo, osFamily string, cweDict map[string]models.CweDictEntry, packs models.Packages) string { +func (w SlackWriter) attachmentText(vinfo models.VulnInfo, cweDict map[string]models.CweDictEntry, packs models.Packages) string { maxCvss := vinfo.MaxCvssScore() vectors := []string{} @@ -277,7 +282,7 @@ func attachmentText(vinfo models.VulnInfo, osFamily string, cweDict map[string]m } else { if 0 < len(vinfo.DistroAdvisories) { links := []string{} - for _, v := range vinfo.CveContents.PrimarySrcURLs(config.Conf.Lang, osFamily, vinfo.CveID) { + for _, v := range vinfo.CveContents.PrimarySrcURLs(w.lang, w.osFamily, vinfo.CveID) { links = append(links, fmt.Sprintf("<%s|%s>", v.Value, v.Type)) } @@ -312,16 +317,16 @@ func attachmentText(vinfo models.VulnInfo, osFamily string, cweDict map[string]m nwvec, vinfo.PatchStatus(packs), strings.Join(vectors, "\n"), - vinfo.Summaries(config.Conf.Lang, osFamily)[0].Value, + vinfo.Summaries(w.lang, w.osFamily)[0].Value, mitigation, - cweIDs(vinfo, osFamily, cweDict), + w.cweIDs(vinfo, w.osFamily, cweDict), ) } -func cweIDs(vinfo models.VulnInfo, osFamily string, cweDict models.CweDict) string { +func (w SlackWriter) cweIDs(vinfo models.VulnInfo, osFamily string, cweDict models.CweDict) string { links := []string{} for _, c := range vinfo.CveContents.UniqCweIDs(osFamily) { - name, url, top10Rank, top10URL, cweTop25Rank, cweTop25URL, sansTop25Rank, sansTop25URL := cweDict.Get(c.Value, osFamily) + name, url, top10Rank, top10URL, cweTop25Rank, cweTop25URL, sansTop25Rank, sansTop25URL := cweDict.Get(c.Value, w.lang) line := "" if top10Rank != "" { line = fmt.Sprintf("<%s|[OWASP Top %s]>", @@ -344,7 +349,7 @@ func cweIDs(vinfo models.VulnInfo, osFamily string, cweDict models.CweDict) stri } // See testcase -func getNotifyUsers(notifyUsers []string) string { +func (w SlackWriter) getNotifyUsers(notifyUsers []string) string { slackStyleTexts := []string{} for _, username := range notifyUsers { slackStyleTexts = append(slackStyleTexts, fmt.Sprintf("<%s>", username)) diff --git a/report/slack_test.go b/reporter/slack_test.go similarity index 83% rename from report/slack_test.go rename to reporter/slack_test.go index 0eae9031..3c587299 100644 --- a/report/slack_test.go +++ b/reporter/slack_test.go @@ -1,4 +1,4 @@ -package report +package reporter import "testing" @@ -14,7 +14,7 @@ func TestGetNotifyUsers(t *testing.T) { } for _, tt := range tests { - actual := getNotifyUsers(tt.in) + actual := SlackWriter{}.getNotifyUsers(tt.in) if tt.expected != actual { t.Errorf("expected %s, actual %s", tt.expected, actual) } diff --git a/report/stdout.go b/reporter/stdout.go similarity index 73% rename from report/stdout.go rename to reporter/stdout.go index 01bf1f7d..86be33c4 100644 --- a/report/stdout.go +++ b/reporter/stdout.go @@ -1,14 +1,20 @@ -package report +package reporter import ( "fmt" - c "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" ) // StdoutWriter write to stdout -type StdoutWriter struct{} +type StdoutWriter struct { + FormatCsv bool + FormatFullText bool + FormatOneLineText bool + FormatList bool +} + +//TODO support -format-jSON // WriteScanSummary prints Scan summary at the end of scan func (w StdoutWriter) WriteScanSummary(rs ...models.ScanResult) { @@ -19,7 +25,7 @@ func (w StdoutWriter) WriteScanSummary(rs ...models.ScanResult) { } func (w StdoutWriter) Write(rs ...models.ScanResult) error { - if c.Conf.FormatOneLineText { + if w.FormatOneLineText { fmt.Print("\n\n") fmt.Println("One Line Summary") fmt.Println("================") @@ -27,13 +33,13 @@ func (w StdoutWriter) Write(rs ...models.ScanResult) error { fmt.Print("\n") } - if c.Conf.FormatList || c.Conf.FormatCsvList { + if w.FormatList || w.FormatCsv { for _, r := range rs { fmt.Println(formatList(r)) } } - if c.Conf.FormatFullText { + if w.FormatFullText { for _, r := range rs { fmt.Println(formatFullPlainText(r)) } diff --git a/report/syslog.go b/reporter/syslog.go similarity index 99% rename from report/syslog.go rename to reporter/syslog.go index 057f2dbf..d02271d7 100644 --- a/report/syslog.go +++ b/reporter/syslog.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "fmt" diff --git a/report/syslog_test.go b/reporter/syslog_test.go similarity index 99% rename from report/syslog_test.go rename to reporter/syslog_test.go index b0f14a8d..f2990752 100644 --- a/report/syslog_test.go +++ b/reporter/syslog_test.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "sort" diff --git a/report/telegram.go b/reporter/telegram.go similarity index 95% rename from report/telegram.go rename to reporter/telegram.go index ec917e4a..c47425cb 100644 --- a/report/telegram.go +++ b/reporter/telegram.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "bytes" @@ -25,7 +25,7 @@ func (w TelegramWriter) Write(rs ...models.ScanResult) (err error) { r.ServerInfo(), r.ScannedCves.FormatCveSummary(), r.ScannedCves.FormatFixedStatus(r.Packages), - r.FormatUpdatablePacksSummary())} + r.FormatUpdatablePkgsSummary())} for _, vinfo := range r.ScannedCves { maxCvss := vinfo.MaxCvssScore() severity := strings.ToUpper(maxCvss.Value.Severity) @@ -38,7 +38,7 @@ func (w TelegramWriter) Write(rs ...models.ScanResult) (err error) { strconv.FormatFloat(maxCvss.Value.Score, 'f', 1, 64), severity, maxCvss.Value.Vector, - vinfo.Summaries(config.Conf.Lang, r.Family)[0].Value)) + vinfo.Summaries(r.Lang, r.Family)[0].Value)) if len(msgs) == 5 { if err = sendMessage(conf.ChatID, conf.Token, strings.Join(msgs, "\n\n")); err != nil { return err diff --git a/report/util.go b/reporter/util.go similarity index 86% rename from report/util.go rename to reporter/util.go index f4b95b94..8f0d417d 100644 --- a/report/util.go +++ b/reporter/util.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "bytes" @@ -8,7 +8,6 @@ import ( "io/ioutil" "os" "path/filepath" - "reflect" "regexp" "sort" "strings" @@ -23,9 +22,7 @@ import ( ) const ( - vulsOpenTag = "" - vulsCloseTag = "" - maxColWidth = 100 + maxColWidth = 100 ) func formatScanSummary(rs ...models.ScanResult) string { @@ -40,7 +37,7 @@ func formatScanSummary(rs ...models.ScanResult) string { cols = []interface{}{ r.FormatServerName(), fmt.Sprintf("%s%s", r.Family, r.Release), - r.FormatUpdatablePacksSummary(), + r.FormatUpdatablePkgsSummary(), } if 0 < len(r.WordPressPackages) { cols = append(cols, fmt.Sprintf("%d WordPress pkgs", len(r.WordPressPackages))) @@ -79,7 +76,7 @@ func formatOneLineSummary(rs ...models.ScanResult) string { r.FormatServerName(), r.ScannedCves.FormatCveSummary(), r.ScannedCves.FormatFixedStatus(r.Packages), - r.FormatUpdatablePacksSummary(), + r.FormatUpdatablePkgsSummary(), r.FormatExploitCveSummary(), r.FormatMetasploitCveSummary(), r.FormatAlertSummary(), @@ -124,7 +121,7 @@ func formatList(r models.ScanResult) string { %s No CVE-IDs are found in updatable packages. %s -`, header, r.FormatUpdatablePacksSummary()) +`, header, r.FormatUpdatablePkgsSummary()) } data := [][]string{} @@ -149,7 +146,7 @@ No CVE-IDs are found in updatable packages. } data = append(data, []string{ - vinfo.CveIDDiffFormat(config.Conf.DiffMinus || config.Conf.DiffPlus), + vinfo.CveIDDiffFormat(), fmt.Sprintf("%4.1f", max), fmt.Sprintf("%5s", vinfo.AttackVector()), // fmt.Sprintf("%4.1f", v2max), @@ -199,7 +196,7 @@ func formatFullPlainText(r models.ScanResult) (lines string) { %s No CVE-IDs are found in updatable packages. %s -`, header, r.FormatUpdatablePacksSummary()) +`, header, r.FormatUpdatablePkgsSummary()) } lines = header + "\n" @@ -220,14 +217,13 @@ No CVE-IDs are found in updatable packages. } data = append(data, []string{"Summary", vuln.Summaries( - config.Conf.Lang, r.Family)[0].Value}) + r.Lang, r.Family)[0].Value}) for _, m := range vuln.Mitigations { data = append(data, []string{"Mitigation", m.URL}) } - links := vuln.CveContents.PrimarySrcURLs( - config.Conf.Lang, r.Family, vuln.CveID) + links := vuln.CveContents.PrimarySrcURLs(r.Lang, r.Family, vuln.CveID) for _, link := range links { data = append(data, []string{"Primary Src", link.Value}) } @@ -373,7 +369,7 @@ No CVE-IDs are found in updatable packages. table.SetColWidth(80) table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) table.SetHeader([]string{ - vuln.CveIDDiffFormat(config.Conf.DiffMinus || config.Conf.DiffPlus), + vuln.CveIDDiffFormat(), vuln.PatchStatus(r.Packages), }) table.SetBorder(true) @@ -432,97 +428,17 @@ func cweJvnURL(cweID string) string { return fmt.Sprintf("http://jvndb.jvn.jp/ja/cwe/%s.html", cweID) } -func formatChangelogs(r models.ScanResult) string { - buf := []string{} - for _, p := range r.Packages { - if p.NewVersion == "" { - continue - } - clog := p.FormatChangelog() - buf = append(buf, clog, "\n\n") +func OverwriteJSONFile(dir string, r models.ScanResult) error { + w := LocalFileWriter{ + CurrentDir: dir, + FormatJSON: true, } - return strings.Join(buf, "\n") -} - -func reuseScannedCves(r *models.ScanResult) bool { - switch r.Family { - case - config.FreeBSD, - config.Raspbian: - return true - } - if isTrivyResult(r) { - return true - } - return false -} - -func isTrivyResult(r *models.ScanResult) bool { - _, ok := r.Optional["trivy-target"] - return ok -} - -func needToRefreshCve(r models.ScanResult) bool { - if r.Lang != config.Conf.Lang { - return true - } - - for _, cve := range r.ScannedCves { - if 0 < len(cve.CveContents) { - return false - } - } - return true -} - -func overwriteJSONFile(dir string, r models.ScanResult) error { - before := config.Conf.FormatJSON - beforePlusDiff := config.Conf.DiffPlus - beforeMinusDiff := config.Conf.DiffMinus - config.Conf.FormatJSON = true - config.Conf.DiffPlus = false - config.Conf.DiffMinus = false - w := LocalFileWriter{CurrentDir: dir} if err := w.Write(r); err != nil { return xerrors.Errorf("Failed to write summary report: %w", err) } - config.Conf.FormatJSON = before - config.Conf.DiffPlus = beforePlusDiff - config.Conf.DiffMinus = beforeMinusDiff return nil } -func loadPrevious(currs models.ScanResults) (prevs models.ScanResults, err error) { - dirs, err := ListValidJSONDirs() - if err != nil { - return - } - - for _, result := range currs { - filename := result.ServerName + ".json" - if result.Container.Name != "" { - filename = fmt.Sprintf("%s@%s.json", result.Container.Name, result.ServerName) - } - for _, dir := range dirs[1:] { - path := filepath.Join(dir, filename) - r, err := loadOneServerScanResult(path) - if err != nil { - util.Log.Debugf("%+v", err) - continue - } - if r.Family == result.Family && r.Release == result.Release { - prevs = append(prevs, *r) - util.Log.Infof("Previous json found: %s", path) - break - } else { - util.Log.Infof("Previous json is different family.Release: %s, pre: %s.%s cur: %s.%s", - path, r.Family, r.Release, result.Family, result.Release) - } - } - } - return prevs, nil -} - func diff(curResults, preResults models.ScanResults, isPlus, isMinus bool) (diffed models.ScanResults) { for _, current := range curResults { found := false @@ -637,21 +553,6 @@ func getMinusDiffCves(previous, current models.ScanResult) models.VulnInfos { return clear } -func isCveFixed(current models.VulnInfo, previous models.ScanResult) bool { - preVinfo, _ := previous.ScannedCves[current.CveID] - pre := map[string]bool{} - for _, h := range preVinfo.AffectedPackages { - pre[h.Name] = h.NotFixedYet - } - - cur := map[string]bool{} - for _, h := range current.AffectedPackages { - cur[h.Name] = h.NotFixedYet - } - - return !reflect.DeepEqual(pre, cur) -} - func isCveInfoUpdated(cveID string, previous, current models.ScanResult) bool { cTypes := []models.CveContentType{ models.Nvd, diff --git a/report/util_test.go b/reporter/util_test.go similarity index 87% rename from report/util_test.go rename to reporter/util_test.go index 651b75b3..600d54e3 100644 --- a/report/util_test.go +++ b/reporter/util_test.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "os" @@ -630,104 +630,3 @@ func TestMinusDiff(t *testing.T) { } } } - -func TestIsCveFixed(t *testing.T) { - type In struct { - v models.VulnInfo - prev models.ScanResult - } - var tests = []struct { - in In - expected bool - }{ - { - in: In{ - v: models.VulnInfo{ - CveID: "CVE-2016-6662", - AffectedPackages: models.PackageFixStatuses{ - { - Name: "mysql-libs", - NotFixedYet: false, - }, - }, - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.Nvd, - CveID: "CVE-2016-6662", - LastModified: time.Time{}, - }, - ), - }, - prev: models.ScanResult{ - ScannedCves: models.VulnInfos{ - "CVE-2016-6662": { - CveID: "CVE-2016-6662", - AffectedPackages: models.PackageFixStatuses{ - { - Name: "mysql-libs", - NotFixedYet: true, - }, - }, - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.Nvd, - CveID: "CVE-2016-6662", - LastModified: time.Time{}, - }, - ), - }, - }, - }, - }, - expected: true, - }, - { - in: In{ - v: models.VulnInfo{ - CveID: "CVE-2016-6662", - AffectedPackages: models.PackageFixStatuses{ - { - Name: "mysql-libs", - NotFixedYet: true, - }, - }, - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.Nvd, - CveID: "CVE-2016-6662", - LastModified: time.Time{}, - }, - ), - }, - prev: models.ScanResult{ - ScannedCves: models.VulnInfos{ - "CVE-2016-6662": { - CveID: "CVE-2016-6662", - AffectedPackages: models.PackageFixStatuses{ - { - Name: "mysql-libs", - NotFixedYet: true, - }, - }, - CveContents: models.NewCveContents( - models.CveContent{ - Type: models.Nvd, - CveID: "CVE-2016-6662", - LastModified: time.Time{}, - }, - ), - }, - }, - }, - }, - expected: false, - }, - } - - for i, tt := range tests { - actual := isCveFixed(tt.in.v, tt.in.prev) - if actual != tt.expected { - t.Errorf("[%d] actual: %t, expected: %t", i, actual, tt.expected) - } - } -} diff --git a/report/writer.go b/reporter/writer.go similarity index 96% rename from report/writer.go rename to reporter/writer.go index d9c64a7b..698e0b7b 100644 --- a/report/writer.go +++ b/reporter/writer.go @@ -1,4 +1,4 @@ -package report +package reporter import ( "bytes" diff --git a/scan/utils.go b/scan/utils.go deleted file mode 100644 index 5cb98f6d..00000000 --- a/scan/utils.go +++ /dev/null @@ -1,36 +0,0 @@ -package scan - -import ( - "fmt" - "strings" - - "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/models" - "github.com/future-architect/vuls/util" -) - -func isRunningKernel(pack models.Package, family string, kernel models.Kernel) (isKernel, running bool) { - switch family { - case config.SUSEEnterpriseServer: - if pack.Name == "kernel-default" { - // Remove the last period and later because uname don't show that. - ss := strings.Split(pack.Release, ".") - rel := strings.Join(ss[0:len(ss)-1], ".") - ver := fmt.Sprintf("%s-%s-default", pack.Version, rel) - return true, kernel.Release == ver - } - return false, false - - case config.RedHat, config.Oracle, config.CentOS, config.Amazon: - switch pack.Name { - case "kernel", "kernel-devel": - ver := fmt.Sprintf("%s-%s.%s", pack.Version, pack.Release, pack.Arch) - return true, kernel.Release == ver - } - return false, false - - default: - util.Log.Warnf("Reboot required is not implemented yet: %s, %v", family, kernel) - } - return false, false -} diff --git a/scan/alpine.go b/scanner/alpine.go similarity index 97% rename from scan/alpine.go rename to scanner/alpine.go index f90a6fa3..0ad06bb1 100644 --- a/scan/alpine.go +++ b/scanner/alpine.go @@ -1,10 +1,11 @@ -package scan +package scanner import ( "bufio" "strings" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" "golang.org/x/xerrors" @@ -38,7 +39,7 @@ func detectAlpine(c config.ServerInfo) (bool, osTypeInterface) { } if r := exec(c, "cat /etc/alpine-release", noSudo); r.isSuccess() { os := newAlpine(c) - os.setDistro(config.Alpine, strings.TrimSpace(r.Stdout)) + os.setDistro(constant.Alpine, strings.TrimSpace(r.Stdout)) return true, os } return false, nil diff --git a/scan/alpine_test.go b/scanner/alpine_test.go similarity index 98% rename from scan/alpine_test.go rename to scanner/alpine_test.go index dc7fbaa5..c4e9df8d 100644 --- a/scan/alpine_test.go +++ b/scanner/alpine_test.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "reflect" diff --git a/scan/amazon.go b/scanner/amazon.go similarity index 99% rename from scan/amazon.go rename to scanner/amazon.go index daf60340..3f0d1b70 100644 --- a/scan/amazon.go +++ b/scanner/amazon.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "github.com/future-architect/vuls/config" diff --git a/scan/base.go b/scanner/base.go similarity index 97% rename from scan/base.go rename to scanner/base.go index 17fbb275..bad1a488 100644 --- a/scan/base.go +++ b/scanner/base.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "bufio" @@ -14,6 +14,7 @@ import ( "github.com/aquasecurity/fanal/analyzer" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" "github.com/sirupsen/logrus" @@ -42,6 +43,24 @@ type base struct { warns []error } +// osPackages is included by base struct +type osPackages struct { + // installed packages + Packages models.Packages + + // installed source packages (Debian based only) + SrcPackages models.SrcPackages + + // enabled dnf modules or packages + EnabledDnfModules []string + + // unsecure packages + VulnInfos models.VulnInfos + + // kernel information + Kernel models.Kernel +} + func (l *base) exec(cmd string, sudo bool) execResult { return exec(l.ServerInfo, cmd, sudo, l.log) } @@ -86,7 +105,7 @@ func (l *base) runningKernel() (release, version string, err error) { release = strings.TrimSpace(r.Stdout) switch l.Distro.Family { - case config.Debian: + case constant.Debian: r := l.exec("uname -a", noSudo) if !r.isSuccess() { return "", "", xerrors.Errorf("Failed to SSH: %s", r) @@ -338,13 +357,12 @@ func (l *base) detectDeepSecurity() (string, error) { return "", xerrors.Errorf("Failed to detect deepsecurity %s", l.ServerInfo.ServerName) } -func (l *base) detectIPSs() { +func (l *base) detectIPS() { if !config.Conf.DetectIPS { return } - ips := map[config.IPS]string{} - + ips := map[string]string{} fingerprint, err := l.detectDeepSecurity() if err != nil { return @@ -428,7 +446,7 @@ func (l *base) convertToModel() models.ScanResult { scannedVia := scannedViaRemote if isLocalExec(l.ServerInfo.Port, l.ServerInfo.Host) { scannedVia = scannedViaLocal - } else if l.ServerInfo.Type == config.ServerTypePseudo { + } else if l.ServerInfo.Type == constant.ServerTypePseudo { scannedVia = scannedViaPseudo } @@ -568,7 +586,7 @@ func (l *base) scanLibraries() (err error) { var bytes []byte switch l.Distro.Family { - case config.ServerTypePseudo: + case constant.ServerTypePseudo: bytes, err = ioutil.ReadFile(path) if err != nil { return xerrors.Errorf("Failed to get target file: %s, filepath: %s", err, path) @@ -622,7 +640,7 @@ func (d *DummyFileInfo) IsDir() bool { return false } func (d *DummyFileInfo) Sys() interface{} { return nil } func (l *base) scanWordPress() error { - if l.ServerInfo.WordPress.IsZero() || l.ServerInfo.Type == config.ServerTypePseudo { + if l.ServerInfo.WordPress.IsZero() || l.ServerInfo.Type == constant.ServerTypePseudo { return nil } l.log.Info("Scanning WordPress...") diff --git a/scan/base_test.go b/scanner/base_test.go similarity index 99% rename from scan/base_test.go rename to scanner/base_test.go index 4d9c5aa2..e8790a5c 100644 --- a/scan/base_test.go +++ b/scanner/base_test.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "reflect" diff --git a/scan/centos.go b/scanner/centos.go similarity index 99% rename from scan/centos.go rename to scanner/centos.go index 7a44725d..159545ec 100644 --- a/scan/centos.go +++ b/scanner/centos.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "github.com/future-architect/vuls/config" diff --git a/scan/debian.go b/scanner/debian.go similarity index 98% rename from scan/debian.go rename to scanner/debian.go index ec8401de..ed14d4c7 100644 --- a/scan/debian.go +++ b/scanner/debian.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "bufio" @@ -12,6 +12,7 @@ import ( "github.com/future-architect/vuls/cache" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" version "github.com/knqyf263/go-deb-version" @@ -59,7 +60,7 @@ func detectDebian(c config.ServerInfo) (bool, osTypeInterface, error) { // e.g. // Raspbian GNU/Linux 7 \n \l result := strings.Fields(r.Stdout) - if len(result) > 2 && result[0] == config.Raspbian { + if len(result) > 2 && result[0] == constant.Raspbian { deb := newDebian(c) deb.setDistro(strings.ToLower(trim(result[0])), trim(result[2])) return true, deb, nil @@ -109,7 +110,7 @@ func detectDebian(c config.ServerInfo) (bool, osTypeInterface, error) { cmd := "cat /etc/debian_version" if r := exec(c, cmd, noSudo); r.isSuccess() { deb := newDebian(c) - deb.setDistro(config.Debian, trim(r.Stdout)) + deb.setDistro(constant.Debian, trim(r.Stdout)) return true, deb, nil } @@ -186,7 +187,7 @@ func (o *debian) checkDeps() error { }) } - if o.Distro.Family == config.Debian { + if o.Distro.Family == constant.Debian { // https://askubuntu.com/a/742844 if !o.ServerInfo.IsContainer() { deps = append(deps, dep{ @@ -305,7 +306,7 @@ func (o *debian) scanPackages() error { return nil } - if !o.getServerInfo().Mode.IsDeep() && o.Distro.Family == config.Raspbian { + if !o.getServerInfo().Mode.IsDeep() && o.Distro.Family == constant.Raspbian { raspbianPacks := o.grepRaspbianPackages(updatable) unsecures, err := o.scanUnsecurePackages(raspbianPacks) if err != nil { @@ -502,7 +503,7 @@ func (o *debian) scanUnsecurePackages(updatable models.Packages) (models.VulnInf // Make a directory for saving changelog to get changelog in Raspbian tmpClogPath := "" - if o.Distro.Family == config.Raspbian { + if o.Distro.Family == constant.Raspbian { tmpClogPath, err = o.makeTempChangelogDir() if err != nil { return nil, err @@ -516,7 +517,7 @@ func (o *debian) scanUnsecurePackages(updatable models.Packages) (models.VulnInf } // Delete a directory for saving changelog to get changelog in Raspbian - if o.Distro.Family == config.Raspbian { + if o.Distro.Family == constant.Raspbian { err := o.deleteTempChangelogDir(tmpClogPath) if err != nil { return nil, xerrors.Errorf("Failed to delete directory to save changelog for Raspbian. err: %s", err) @@ -820,11 +821,11 @@ func (o *debian) fetchParseChangelog(pack models.Package, tmpClogPath string) ([ cmd := "" switch o.Distro.Family { - case config.Ubuntu: + case constant.Ubuntu: cmd = fmt.Sprintf(`PAGER=cat apt-get -q=2 changelog %s`, pack.Name) - case config.Debian: + case constant.Debian: cmd = fmt.Sprintf(`PAGER=cat aptitude -q=2 changelog %s`, pack.Name) - case config.Raspbian: + case constant.Raspbian: changelogPath, err := o.getChangelogPath(pack.Name, tmpClogPath) if err != nil { // Ignore this Error. @@ -936,10 +937,10 @@ func (o *debian) getCveIDsFromChangelog( delim := []string{"+", "~", "build"} switch o.Distro.Family { - case config.Ubuntu: - delim = append(delim, config.Ubuntu) - case config.Debian: - case config.Raspbian: + case constant.Ubuntu: + delim = append(delim, constant.Ubuntu) + case constant.Debian: + case constant.Raspbian: } for _, d := range delim { @@ -1014,7 +1015,7 @@ func (o *debian) parseChangelog(changelog, name, ver string, confidence models.C } if !found { - if o.Distro.Family == config.Raspbian { + if o.Distro.Family == constant.Raspbian { pack := o.Packages[name] pack.Changelog = &models.Changelog{ Contents: strings.Join(buf, "\n"), diff --git a/scan/debian_test.go b/scanner/debian_test.go similarity index 99% rename from scan/debian_test.go rename to scanner/debian_test.go index aa429eb7..f02a4ba7 100644 --- a/scan/debian_test.go +++ b/scanner/debian_test.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "os" @@ -8,6 +8,7 @@ import ( "github.com/future-architect/vuls/cache" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/k0kubun/pp" "github.com/sirupsen/logrus" @@ -851,7 +852,7 @@ vlc (3.0.11-0+deb10u1) buster-security; urgency=high } o := newDebian(config.ServerInfo{}) - o.Distro = config.Distro{Family: config.Raspbian} + o.Distro = config.Distro{Family: constant.Raspbian} for _, tt := range tests { t.Run(tt.packName, func(t *testing.T) { cveIDs, pack, _ := o.parseChangelog(tt.args.changelog, tt.args.name, tt.args.ver, models.ChangelogExactMatch) diff --git a/scan/executil.go b/scanner/executil.go similarity index 62% rename from scan/executil.go rename to scanner/executil.go index df215a92..9cfac34b 100644 --- a/scan/executil.go +++ b/scanner/executil.go @@ -1,24 +1,16 @@ -package scan +package scanner import ( "bytes" - "crypto/x509" - "encoding/pem" "fmt" - "io/ioutil" - "net" - "os" ex "os/exec" "path/filepath" "strings" "syscall" "time" - "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/agent" "golang.org/x/xerrors" - "github.com/cenkalti/backoff" conf "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/util" homedir "github.com/mitchellh/go-homedir" @@ -147,8 +139,6 @@ func exec(c conf.ServerInfo, cmd string, sudo bool, log ...*logrus.Entry) (resul if isLocalExec(c.Port, c.Host) { result = localExec(c, cmd, sudo) - } else if conf.Conf.SSHNative { - result = sshExecNative(c, cmd, sudo) } else { result = sshExecExternal(c, cmd, sudo) } @@ -192,70 +182,10 @@ func localExec(c conf.ServerInfo, cmdstr string, sudo bool) (result execResult) return } -func sshExecNative(c conf.ServerInfo, cmd string, sudo bool) (result execResult) { - result.Servername = c.ServerName - result.Container = c.Container - result.Host = c.Host - result.Port = c.Port - - var client *ssh.Client - var err error - if client, err = sshConnect(c); err != nil { - result.Error = err - result.ExitStatus = 999 - return - } - defer client.Close() - - var session *ssh.Session - if session, err = client.NewSession(); err != nil { - result.Error = xerrors.Errorf( - "Failed to create a new session. servername: %s, err: %w", - c.ServerName, err) - result.ExitStatus = 999 - return - } - defer session.Close() - - // http://blog.ralch.com/tutorial/golang-ssh-connection/ - modes := ssh.TerminalModes{ - ssh.ECHO: 0, // disable echoing - ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud - ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud - } - if err = session.RequestPty("xterm", 400, 1000, modes); err != nil { - result.Error = xerrors.Errorf( - "Failed to request for pseudo terminal. servername: %s, err: %w", - c.ServerName, err) - result.ExitStatus = 999 - return - } - - var stdoutBuf, stderrBuf bytes.Buffer - session.Stdout = &stdoutBuf - session.Stderr = &stderrBuf - - cmd = decorateCmd(c, cmd, sudo) - if err := session.Run(cmd); err != nil { - if exitErr, ok := err.(*ssh.ExitError); ok { - result.ExitStatus = exitErr.ExitStatus() - } else { - result.ExitStatus = 999 - } - } else { - result.ExitStatus = 0 - } - - result.Stdout = stdoutBuf.String() - result.Stderr = stderrBuf.String() - result.Cmd = strings.Replace(cmd, "\n", "", -1) - return -} - func sshExecExternal(c conf.ServerInfo, cmd string, sudo bool) (result execResult) { sshBinaryPath, err := ex.LookPath("ssh") if err != nil { - return sshExecNative(c, cmd, sudo) + return execResult{Error: err} } defaultSSHArgs := []string{"-tt"} @@ -383,115 +313,3 @@ func decorateCmd(c conf.ServerInfo, cmd string, sudo bool) string { // cmd = fmt.Sprintf("set -x; %s", cmd) return cmd } - -func getAgentAuth() (auth ssh.AuthMethod, ok bool) { - if sock := os.Getenv("SSH_AUTH_SOCK"); 0 < len(sock) { - if agconn, err := net.Dial("unix", sock); err == nil { - ag := agent.NewClient(agconn) - auth = ssh.PublicKeysCallback(ag.Signers) - ok = true - } - } - return -} - -func tryAgentConnect(c conf.ServerInfo) *ssh.Client { - if auth, ok := getAgentAuth(); ok { - config := &ssh.ClientConfig{ - User: c.User, - Auth: []ssh.AuthMethod{auth}, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - client, _ := ssh.Dial("tcp", c.Host+":"+c.Port, config) - return client - } - return nil -} - -func sshConnect(c conf.ServerInfo) (client *ssh.Client, err error) { - if client = tryAgentConnect(c); client != nil { - return client, nil - } - - var auths = []ssh.AuthMethod{} - if auths, err = addKeyAuth(auths, c.KeyPath, c.KeyPassword); err != nil { - return nil, err - } - - // http://blog.ralch.com/tutorial/golang-ssh-connection/ - config := &ssh.ClientConfig{ - User: c.User, - Auth: auths, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - notifyFunc := func(e error, t time.Duration) { - logger := getSSHLogger() - logger.Debugf("Failed to Dial to %s, err: %s, Retrying in %s...", - c.ServerName, e, t) - } - err = backoff.RetryNotify(func() error { - if client, err = ssh.Dial("tcp", c.Host+":"+c.Port, config); err != nil { - return err - } - return nil - }, backoff.NewExponentialBackOff(), notifyFunc) - - return -} - -// https://github.com/rapidloop/rtop/blob/ba5b35e964135d50e0babedf0bd69b2fcb5dbcb4/src/sshhelper.go#L100 -func addKeyAuth(auths []ssh.AuthMethod, keypath string, keypassword string) ([]ssh.AuthMethod, error) { - if len(keypath) == 0 { - return auths, nil - } - - // read the file - pemBytes, err := ioutil.ReadFile(keypath) - if err != nil { - return auths, err - } - - // get first pem block - block, _ := pem.Decode(pemBytes) - if block == nil { - return auths, xerrors.Errorf("no key found in %s", keypath) - } - - // handle plain and encrypted keyfiles - if x509.IsEncryptedPEMBlock(block) { - block.Bytes, err = x509.DecryptPEMBlock(block, []byte(keypassword)) - if err != nil { - return auths, err - } - key, err := parsePemBlock(block) - if err != nil { - return auths, err - } - signer, err := ssh.NewSignerFromKey(key) - if err != nil { - return auths, err - } - return append(auths, ssh.PublicKeys(signer)), nil - } - - signer, err := ssh.ParsePrivateKey(pemBytes) - if err != nil { - return auths, err - } - return append(auths, ssh.PublicKeys(signer)), nil -} - -// ref golang.org/x/crypto/ssh/keys.go#ParseRawPrivateKey. -func parsePemBlock(block *pem.Block) (interface{}, error) { - switch block.Type { - case "RSA PRIVATE KEY": - return x509.ParsePKCS1PrivateKey(block.Bytes) - case "EC PRIVATE KEY": - return x509.ParseECPrivateKey(block.Bytes) - case "DSA PRIVATE KEY": - return ssh.ParseDSAPrivateKey(block.Bytes) - default: - return nil, xerrors.Errorf("Unsupported key type %q", block.Type) - } -} diff --git a/scan/executil_test.go b/scanner/executil_test.go similarity index 99% rename from scan/executil_test.go rename to scanner/executil_test.go index 71e41a41..4ed33b16 100644 --- a/scan/executil_test.go +++ b/scanner/executil_test.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "testing" diff --git a/scan/freebsd.go b/scanner/freebsd.go similarity index 97% rename from scan/freebsd.go rename to scanner/freebsd.go index 3cd61336..86d1a281 100644 --- a/scan/freebsd.go +++ b/scanner/freebsd.go @@ -1,10 +1,11 @@ -package scan +package scanner import ( "net" "strings" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" "golang.org/x/xerrors" @@ -33,14 +34,14 @@ func newBsd(c config.ServerInfo) *bsd { //https://github.com/mizzy/specinfra/blob/master/lib/specinfra/helper/detect_os/freebsd.rb func detectFreebsd(c config.ServerInfo) (bool, osTypeInterface) { // Prevent from adding `set -o pipefail` option - c.Distro = config.Distro{Family: config.FreeBSD} + c.Distro = config.Distro{Family: constant.FreeBSD} if r := exec(c, "uname", noSudo); r.isSuccess() { - if strings.Contains(strings.ToLower(r.Stdout), config.FreeBSD) == true { + if strings.Contains(strings.ToLower(r.Stdout), constant.FreeBSD) == true { if b := exec(c, "freebsd-version", noSudo); b.isSuccess() { bsd := newBsd(c) rel := strings.TrimSpace(b.Stdout) - bsd.setDistro(config.FreeBSD, rel) + bsd.setDistro(constant.FreeBSD, rel) return true, bsd } } diff --git a/scan/freebsd_test.go b/scanner/freebsd_test.go similarity index 99% rename from scan/freebsd_test.go rename to scanner/freebsd_test.go index d5bc7d95..d9e12163 100644 --- a/scan/freebsd_test.go +++ b/scanner/freebsd_test.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "reflect" diff --git a/scan/library.go b/scanner/library.go similarity index 97% rename from scan/library.go rename to scanner/library.go index 7c8c20c9..ce17d0de 100644 --- a/scan/library.go +++ b/scanner/library.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "github.com/aquasecurity/fanal/types" diff --git a/scan/oracle.go b/scanner/oracle.go similarity index 99% rename from scan/oracle.go rename to scanner/oracle.go index a57080dd..f02a5ddd 100644 --- a/scan/oracle.go +++ b/scanner/oracle.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "github.com/future-architect/vuls/config" diff --git a/scan/pseudo.go b/scanner/pseudo.go similarity index 88% rename from scan/pseudo.go rename to scanner/pseudo.go index 526a56c9..12600401 100644 --- a/scan/pseudo.go +++ b/scanner/pseudo.go @@ -1,7 +1,8 @@ -package scan +package scanner import ( "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" ) @@ -12,9 +13,9 @@ type pseudo struct { } func detectPseudo(c config.ServerInfo) (itsMe bool, pseudo osTypeInterface, err error) { - if c.Type == config.ServerTypePseudo { + if c.Type == constant.ServerTypePseudo { p := newPseudo(c) - p.setDistro(config.ServerTypePseudo, "") + p.setDistro(constant.ServerTypePseudo, "") return true, p, nil } return false, nil, nil diff --git a/scan/redhatbase.go b/scanner/redhatbase.go similarity index 95% rename from scan/redhatbase.go rename to scanner/redhatbase.go index 8046bd0f..c97c446a 100644 --- a/scan/redhatbase.go +++ b/scanner/redhatbase.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "bufio" @@ -7,6 +7,7 @@ import ( "strings" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" "golang.org/x/xerrors" @@ -34,7 +35,7 @@ func detectRedhat(c config.ServerInfo) (bool, osTypeInterface) { ora := newOracle(c) release := result[2] - ora.setDistro(config.Oracle, release) + ora.setDistro(constant.Oracle, release) return true, ora } } @@ -54,7 +55,7 @@ func detectRedhat(c config.ServerInfo) (bool, osTypeInterface) { switch strings.ToLower(result[1]) { case "centos", "centos linux", "centos stream": cent := newCentOS(c) - cent.setDistro(config.CentOS, release) + cent.setDistro(constant.CentOS, release) return true, cent default: util.Log.Warnf("Failed to parse CentOS: %s", r) @@ -79,19 +80,19 @@ func detectRedhat(c config.ServerInfo) (bool, osTypeInterface) { switch strings.ToLower(result[1]) { case "centos", "centos linux": cent := newCentOS(c) - cent.setDistro(config.CentOS, release) + cent.setDistro(constant.CentOS, release) return true, cent default: // RHEL rhel := newRHEL(c) - rhel.setDistro(config.RedHat, release) + rhel.setDistro(constant.RedHat, release) return true, rhel } } } if r := exec(c, "ls /etc/system-release", noSudo); r.isSuccess() { - family := config.Amazon + family := constant.Amazon release := "unknown" if r := exec(c, "cat /etc/system-release", noSudo); r.isSuccess() { if strings.HasPrefix(r.Stdout, "Amazon Linux release 2") { @@ -218,7 +219,7 @@ func (o *redhatBase) scanPackages() (err error) { if o.getServerInfo().Mode.IsOffline() { return nil - } else if o.Distro.Family == config.RedHat { + } else if o.Distro.Family == constant.RedHat { if o.getServerInfo().Mode.IsFast() { return nil } @@ -426,12 +427,12 @@ func (o *redhatBase) parseUpdatablePacksLine(line string) (models.Package, error func (o *redhatBase) isExecYumPS() bool { switch o.Distro.Family { - case config.Oracle, - config.OpenSUSE, - config.OpenSUSELeap, - config.SUSEEnterpriseServer, - config.SUSEEnterpriseDesktop, - config.SUSEOpenstackCloud: + case constant.Oracle, + constant.OpenSUSE, + constant.OpenSUSELeap, + constant.SUSEEnterpriseServer, + constant.SUSEEnterpriseDesktop, + constant.SUSEOpenstackCloud: return false } return !o.getServerInfo().Mode.IsFast() @@ -439,15 +440,15 @@ func (o *redhatBase) isExecYumPS() bool { func (o *redhatBase) isExecNeedsRestarting() bool { switch o.Distro.Family { - case config.OpenSUSE, - config.OpenSUSELeap, - config.SUSEEnterpriseServer, - config.SUSEEnterpriseDesktop, - config.SUSEOpenstackCloud: + case constant.OpenSUSE, + constant.OpenSUSELeap, + constant.SUSEEnterpriseServer, + constant.SUSEEnterpriseDesktop, + constant.SUSEOpenstackCloud: // TODO zypper ps // https://github.com/future-architect/vuls/issues/696 return false - case config.RedHat, config.CentOS, config.Oracle: + case constant.RedHat, constant.CentOS, constant.Oracle: majorVersion, err := o.Distro.MajorVersion() if err != nil || majorVersion < 6 { o.log.Errorf("Not implemented yet: %s, err: %s", o.Distro, err) @@ -594,7 +595,7 @@ func (o *redhatBase) rpmQa() string { const old = `rpm -qa --queryformat "%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{ARCH}\n"` const new = `rpm -qa --queryformat "%{NAME} %{EPOCHNUM} %{VERSION} %{RELEASE} %{ARCH}\n"` switch o.Distro.Family { - case config.SUSEEnterpriseServer: + case constant.SUSEEnterpriseServer: if v, _ := o.Distro.MajorVersion(); v < 12 { return old } @@ -611,7 +612,7 @@ func (o *redhatBase) rpmQf() string { const old = `rpm -qf --queryformat "%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{ARCH}\n" ` const new = `rpm -qf --queryformat "%{NAME} %{EPOCHNUM} %{VERSION} %{RELEASE} %{ARCH}\n" ` switch o.Distro.Family { - case config.SUSEEnterpriseServer: + case constant.SUSEEnterpriseServer: if v, _ := o.Distro.MajorVersion(); v < 12 { return old } @@ -626,7 +627,7 @@ func (o *redhatBase) rpmQf() string { func (o *redhatBase) detectEnabledDnfModules() ([]string, error) { switch o.Distro.Family { - case config.RedHat, config.CentOS: + case constant.RedHat, constant.CentOS: //TODO OracleLinux default: return nil, nil diff --git a/scan/redhatbase_test.go b/scanner/redhatbase_test.go similarity index 99% rename from scan/redhatbase_test.go rename to scanner/redhatbase_test.go index 8d2e1fa3..0b5d0f18 100644 --- a/scan/redhatbase_test.go +++ b/scanner/redhatbase_test.go @@ -1,10 +1,11 @@ -package scan +package scanner import ( "reflect" "testing" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/k0kubun/pp" ) @@ -16,7 +17,7 @@ import ( func TestParseInstalledPackagesLinesRedhat(t *testing.T) { r := newRHEL(config.ServerInfo{}) - r.Distro = config.Distro{Family: config.RedHat} + r.Distro = config.Distro{Family: constant.RedHat} var packagetests = []struct { in string diff --git a/scan/rhel.go b/scanner/rhel.go similarity index 99% rename from scan/rhel.go rename to scanner/rhel.go index be1894c9..926e789c 100644 --- a/scan/rhel.go +++ b/scanner/rhel.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "github.com/future-architect/vuls/config" diff --git a/scan/serverapi.go b/scanner/serverapi.go similarity index 63% rename from scan/serverapi.go rename to scanner/serverapi.go index 647decea..0ef63d7a 100644 --- a/scan/serverapi.go +++ b/scanner/serverapi.go @@ -1,16 +1,15 @@ -package scan +package scanner import ( "fmt" "net/http" "os" - "path/filepath" "time" "github.com/future-architect/vuls/cache" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" - "github.com/future-architect/vuls/report" "github.com/future-architect/vuls/util" "golang.org/x/xerrors" ) @@ -30,14 +29,14 @@ var ( var servers, errServers []osTypeInterface -// Base Interface of redhat, debian, freebsd +// Base Interface type osTypeInterface interface { setServerInfo(config.ServerInfo) getServerInfo() config.ServerInfo setDistro(string, string) getDistro() config.Distro detectPlatform() - detectIPSs() + detectIPS() getPlatform() models.Platform checkScanMode() error @@ -62,58 +61,372 @@ type osTypeInterface interface { setErrs([]error) } -// osPackages is included by base struct -type osPackages struct { - // installed packages - Packages models.Packages +// Scanner has functions for scan +type Scanner struct { + TimeoutSec int + ScanTimeoutSec int + CacheDBPath string - // installed source packages (Debian based only) - SrcPackages models.SrcPackages - - // enabled dnf modules or packages - EnabledDnfModules []string - - // unsecure packages - VulnInfos models.VulnInfos - - // kernel information - Kernel models.Kernel + Targets map[string]config.ServerInfo } -// Retry as it may stall on the first SSH connection -// https://github.com/future-architect/vuls/pull/753 -func detectDebianWithRetry(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err error) { - type Response struct { - itsMe bool - deb osTypeInterface - err error +// Scan execute scan +func (s Scanner) Scan() error { + util.Log.Info("Detecting Server/Container OS... ") + if err := s.initServers(); err != nil { + return xerrors.Errorf("Failed to init servers. err: %w", err) } - resChan := make(chan Response, 1) - go func(c config.ServerInfo) { - itsMe, osType, fatalErr := detectDebian(c) - resChan <- Response{itsMe, osType, fatalErr} - }(c) - timeout := time.After(time.Duration(3) * time.Second) - select { - case res := <-resChan: - return res.itsMe, res.deb, res.err - case <-timeout: - time.Sleep(100 * time.Millisecond) - return detectDebian(c) + util.Log.Info("Checking Scan Modes... ") + if err := s.checkScanModes(); err != nil { + return xerrors.Errorf("Fix config.toml. err: %w", err) } + + util.Log.Info("Detecting Platforms... ") + s.detectPlatform() + + util.Log.Info("Detecting IPS identifiers... ") + s.detectIPS() + + if err := s.execScan(); err != nil { + return xerrors.Errorf("Failed to scan. err: %w", err) + } + return nil } -func detectOS(c config.ServerInfo) (osType osTypeInterface) { +// Configtest checks if the server is scannable. +func (s Scanner) Configtest() error { + util.Log.Info("Detecting Server/Container OS... ") + if err := s.initServers(); err != nil { + return xerrors.Errorf("Failed to init servers. err: %w", err) + } + + util.Log.Info("Checking Scan Modes...") + if err := s.checkScanModes(); err != nil { + return xerrors.Errorf("Fix config.toml. err: %w", err) + } + + util.Log.Info("Checking dependencies...") + s.checkDependencies() + + util.Log.Info("Checking sudo settings...") + s.checkIfSudoNoPasswd() + + util.Log.Info("It can be scanned with fast scan mode even if warn or err messages are displayed due to lack of dependent packages or sudo settings in fast-root or deep scan mode") + + if len(servers) == 0 { + return xerrors.Errorf("No scannable servers") + } + + util.Log.Info("Scannable servers are below...") + for _, s := range servers { + if s.getServerInfo().IsContainer() { + fmt.Printf("%s@%s ", + s.getServerInfo().Container.Name, + s.getServerInfo().ServerName, + ) + } else { + fmt.Printf("%s ", s.getServerInfo().ServerName) + } + } + fmt.Printf("\n") + return nil +} + +// ViaHTTP scans servers by HTTP header and body +func ViaHTTP(header http.Header, body string, toLocalFile bool) (models.ScanResult, error) { + family := header.Get("X-Vuls-OS-Family") + if family == "" { + return models.ScanResult{}, errOSFamilyHeader + } + + release := header.Get("X-Vuls-OS-Release") + if release == "" { + return models.ScanResult{}, errOSReleaseHeader + } + + kernelRelease := header.Get("X-Vuls-Kernel-Release") + if kernelRelease == "" { + util.Log.Warn("If X-Vuls-Kernel-Release is not specified, there is a possibility of false detection") + } + + kernelVersion := header.Get("X-Vuls-Kernel-Version") + if family == constant.Debian && kernelVersion == "" { + return models.ScanResult{}, errKernelVersionHeader + } + + serverName := header.Get("X-Vuls-Server-Name") + if toLocalFile && serverName == "" { + return models.ScanResult{}, errServerNameHeader + } + + distro := config.Distro{ + Family: family, + Release: release, + } + + kernel := models.Kernel{ + Release: kernelRelease, + Version: kernelVersion, + } + base := base{ + Distro: distro, + osPackages: osPackages{ + Kernel: kernel, + }, + log: util.Log, + } + + var osType osTypeInterface + switch family { + case constant.Debian, constant.Ubuntu: + osType = &debian{base: base} + case constant.RedHat: + osType = &rhel{ + redhatBase: redhatBase{base: base}, + } + case constant.CentOS: + osType = ¢os{ + redhatBase: redhatBase{base: base}, + } + case constant.Oracle: + osType = &oracle{ + redhatBase: redhatBase{base: base}, + } + case constant.Amazon: + osType = &amazon{ + redhatBase: redhatBase{base: base}, + } + default: + return models.ScanResult{}, xerrors.Errorf("Server mode for %s is not implemented yet", family) + } + + installedPackages, srcPackages, err := osType.parseInstalledPackages(body) + if err != nil { + return models.ScanResult{}, err + } + + result := models.ScanResult{ + ServerName: serverName, + Family: family, + Release: release, + RunningKernel: models.Kernel{ + Release: kernelRelease, + Version: kernelVersion, + }, + Packages: installedPackages, + SrcPackages: srcPackages, + ScannedCves: models.VulnInfos{}, + } + + return result, nil +} + +// initServers detect the kind of OS distribution of target servers +func (s Scanner) initServers() error { + hosts, errHosts := s.detectServerOSes() + if len(hosts) == 0 { + return xerrors.New("No scannable host OS") + } + containers, errContainers := s.detectContainerOSes(hosts) + + // set to pkg global variable + for _, host := range hosts { + if !host.getServerInfo().ContainersOnly { + servers = append(servers, host) + } + } + servers = append(servers, containers...) + errServers = append(errHosts, errContainers...) + + if len(servers) == 0 { + return xerrors.New("No scannable servers") + } + return nil +} + +func (s Scanner) detectServerOSes() (servers, errServers []osTypeInterface) { + util.Log.Info("Detecting OS of servers... ") + osTypeChan := make(chan osTypeInterface, len(s.Targets)) + defer close(osTypeChan) + for _, target := range s.Targets { + go func(srv config.ServerInfo) { + defer func() { + if p := recover(); p != nil { + util.Log.Debugf("Panic: %s on %s", p, srv.ServerName) + } + }() + osTypeChan <- s.detectOS(srv) + }(target) + } + + timeout := time.After(time.Duration(s.TimeoutSec) * time.Second) + for i := 0; i < len(s.Targets); i++ { + select { + case res := <-osTypeChan: + if 0 < len(res.getErrs()) { + errServers = append(errServers, res) + util.Log.Errorf("(%d/%d) Failed: %s, err: %+v", + i+1, len(s.Targets), res.getServerInfo().ServerName, res.getErrs()) + } else { + servers = append(servers, res) + util.Log.Infof("(%d/%d) Detected: %s: %s", + i+1, len(s.Targets), res.getServerInfo().ServerName, res.getDistro()) + } + case <-timeout: + msg := "Timed out while detecting servers" + util.Log.Error(msg) + for servername, sInfo := range s.Targets { + found := false + for _, o := range append(servers, errServers...) { + if servername == o.getServerInfo().ServerName { + found = true + break + } + } + if !found { + u := &unknown{} + u.setServerInfo(sInfo) + u.setErrs([]error{xerrors.New("Timed out")}) + errServers = append(errServers, u) + util.Log.Errorf("(%d/%d) Timed out: %s", i+1, len(s.Targets), servername) + } + } + } + } + return +} + +func (s Scanner) detectContainerOSes(hosts []osTypeInterface) (actives, inactives []osTypeInterface) { + util.Log.Info("Detecting OS of containers... ") + osTypesChan := make(chan []osTypeInterface, len(hosts)) + defer close(osTypesChan) + for _, host := range hosts { + go func(h osTypeInterface) { + defer func() { + if p := recover(); p != nil { + util.Log.Debugf("Panic: %s on %s", + p, h.getServerInfo().GetServerName()) + } + }() + osTypesChan <- s.detectContainerOSesOnServer(h) + }(host) + } + + timeout := time.After(time.Duration(s.TimeoutSec) * time.Second) + for i := 0; i < len(hosts); i++ { + select { + case res := <-osTypesChan: + for _, osi := range res { + sinfo := osi.getServerInfo() + if 0 < len(osi.getErrs()) { + inactives = append(inactives, osi) + util.Log.Errorf("Failed: %s err: %+v", sinfo.ServerName, osi.getErrs()) + continue + } + actives = append(actives, osi) + util.Log.Infof("Detected: %s@%s: %s", + sinfo.Container.Name, sinfo.ServerName, osi.getDistro()) + } + case <-timeout: + util.Log.Error("Some containers timed out") + } + } + return +} + +func (s Scanner) detectContainerOSesOnServer(containerHost osTypeInterface) (oses []osTypeInterface) { + containerHostInfo := containerHost.getServerInfo() + if len(containerHostInfo.ContainersIncluded) == 0 { + return + } + + running, err := containerHost.runningContainers() + if err != nil { + containerHost.setErrs([]error{xerrors.Errorf( + "Failed to get running containers on %s. err: %w", + containerHost.getServerInfo().ServerName, err)}) + return append(oses, containerHost) + } + + if containerHostInfo.ContainersIncluded[0] == "${running}" { + for _, containerInfo := range running { + found := false + for _, ex := range containerHost.getServerInfo().ContainersExcluded { + if containerInfo.Name == ex || containerInfo.ContainerID == ex { + found = true + } + } + if found { + continue + } + + copied := containerHostInfo + copied.SetContainer(config.Container{ + ContainerID: containerInfo.ContainerID, + Name: containerInfo.Name, + Image: containerInfo.Image, + }) + os := s.detectOS(copied) + oses = append(oses, os) + } + return oses + } + + exitedContainers, err := containerHost.exitedContainers() + if err != nil { + containerHost.setErrs([]error{xerrors.Errorf( + "Failed to get exited containers on %s. err: %w", + containerHost.getServerInfo().ServerName, err)}) + return append(oses, containerHost) + } + + var exited, unknown []string + for _, container := range containerHostInfo.ContainersIncluded { + found := false + for _, c := range running { + if c.ContainerID == container || c.Name == container { + copied := containerHostInfo + copied.SetContainer(c) + os := s.detectOS(copied) + oses = append(oses, os) + found = true + break + } + } + + if !found { + foundInExitedContainers := false + for _, c := range exitedContainers { + if c.ContainerID == container || c.Name == container { + exited = append(exited, container) + foundInExitedContainers = true + break + } + } + if !foundInExitedContainers { + unknown = append(unknown, container) + } + } + } + if 0 < len(exited) || 0 < len(unknown) { + containerHost.setErrs([]error{xerrors.Errorf( + "Some containers on %s are exited or unknown. exited: %s, unknown: %s", + containerHost.getServerInfo().ServerName, exited, unknown)}) + return append(oses, containerHost) + } + return oses +} + +func (s Scanner) detectOS(c config.ServerInfo) (osType osTypeInterface) { var itsMe bool var fatalErr error if itsMe, osType, _ = detectPseudo(c); itsMe { - util.Log.Debugf("Pseudo") return } - itsMe, osType, fatalErr = detectDebianWithRetry(c) + itsMe, osType, fatalErr = s.detectDebianWithRetry(c) if fatalErr != nil { osType.setErrs([]error{ xerrors.Errorf("Failed to detect OS: %w", fatalErr)}) @@ -150,252 +463,32 @@ func detectOS(c config.ServerInfo) (osType osTypeInterface) { return } -// PrintSSHableServerNames print SSH-able servernames -func PrintSSHableServerNames() bool { - if len(servers) == 0 { - util.Log.Error("No scannable servers") - return false +// Retry as it may stall on the first SSH connection +// https://github.com/future-architect/vuls/pull/753 +func (s Scanner) detectDebianWithRetry(c config.ServerInfo) (itsMe bool, deb osTypeInterface, err error) { + type Response struct { + itsMe bool + deb osTypeInterface + err error } - util.Log.Info("Scannable servers are below...") - for _, s := range servers { - if s.getServerInfo().IsContainer() { - fmt.Printf("%s@%s ", - s.getServerInfo().Container.Name, - s.getServerInfo().ServerName, - ) - } else { - fmt.Printf("%s ", s.getServerInfo().ServerName) - } + resChan := make(chan Response, 1) + go func(c config.ServerInfo) { + itsMe, osType, fatalErr := detectDebian(c) + resChan <- Response{itsMe, osType, fatalErr} + }(c) + + timeout := time.After(time.Duration(3) * time.Second) + select { + case res := <-resChan: + return res.itsMe, res.deb, res.err + case <-timeout: + time.Sleep(100 * time.Millisecond) + return detectDebian(c) } - fmt.Printf("\n") - return true } -// InitServers detect the kind of OS distribution of target servers -func InitServers(timeoutSec int) error { - hosts, errHosts := detectServerOSes(timeoutSec) - if len(hosts) == 0 { - return xerrors.New("No scannable host OS") - } - containers, errContainers := detectContainerOSes(hosts, timeoutSec) - - // set to pkg global variable - for _, host := range hosts { - if !host.getServerInfo().ContainersOnly { - servers = append(servers, host) - } - } - servers = append(servers, containers...) - errServers = append(errHosts, errContainers...) - - if len(servers) == 0 { - return xerrors.New("No scannable servers") - } - return nil -} - -func detectServerOSes(timeoutSec int) (servers, errServers []osTypeInterface) { - util.Log.Info("Detecting OS of servers... ") - osTypeChan := make(chan osTypeInterface, len(config.Conf.Servers)) - defer close(osTypeChan) - for _, s := range config.Conf.Servers { - go func(s config.ServerInfo) { - defer func() { - if p := recover(); p != nil { - util.Log.Debugf("Panic: %s on %s", p, s.ServerName) - } - }() - osTypeChan <- detectOS(s) - }(s) - } - - timeout := time.After(time.Duration(timeoutSec) * time.Second) - for i := 0; i < len(config.Conf.Servers); i++ { - select { - case res := <-osTypeChan: - if 0 < len(res.getErrs()) { - errServers = append(errServers, res) - util.Log.Errorf("(%d/%d) Failed: %s, err: %+v", - i+1, len(config.Conf.Servers), - res.getServerInfo().ServerName, - res.getErrs()) - } else { - servers = append(servers, res) - util.Log.Infof("(%d/%d) Detected: %s: %s", - i+1, len(config.Conf.Servers), - res.getServerInfo().ServerName, - res.getDistro()) - } - case <-timeout: - msg := "Timed out while detecting servers" - util.Log.Error(msg) - for servername, sInfo := range config.Conf.Servers { - found := false - for _, o := range append(servers, errServers...) { - if servername == o.getServerInfo().ServerName { - found = true - break - } - } - if !found { - u := &unknown{} - u.setServerInfo(sInfo) - u.setErrs([]error{ - xerrors.New("Timed out"), - }) - errServers = append(errServers, u) - util.Log.Errorf("(%d/%d) Timed out: %s", - i+1, len(config.Conf.Servers), - servername) - i++ - } - } - } - } - return -} - -func detectContainerOSes(hosts []osTypeInterface, timeoutSec int) (actives, inactives []osTypeInterface) { - util.Log.Info("Detecting OS of containers... ") - osTypesChan := make(chan []osTypeInterface, len(hosts)) - defer close(osTypesChan) - for _, s := range hosts { - go func(s osTypeInterface) { - defer func() { - if p := recover(); p != nil { - util.Log.Debugf("Panic: %s on %s", - p, s.getServerInfo().GetServerName()) - } - }() - osTypesChan <- detectContainerOSesOnServer(s) - }(s) - } - - timeout := time.After(time.Duration(timeoutSec) * time.Second) - for i := 0; i < len(hosts); i++ { - select { - case res := <-osTypesChan: - for _, osi := range res { - sinfo := osi.getServerInfo() - if 0 < len(osi.getErrs()) { - inactives = append(inactives, osi) - util.Log.Errorf("Failed: %s err: %+v", sinfo.ServerName, osi.getErrs()) - continue - } - actives = append(actives, osi) - util.Log.Infof("Detected: %s@%s: %s", - sinfo.Container.Name, sinfo.ServerName, osi.getDistro()) - } - case <-timeout: - msg := "Timed out while detecting containers" - util.Log.Error(msg) - for servername, sInfo := range config.Conf.Servers { - found := false - for _, o := range append(actives, inactives...) { - if servername == o.getServerInfo().ServerName { - found = true - break - } - } - if !found { - u := &unknown{} - u.setServerInfo(sInfo) - u.setErrs([]error{ - xerrors.New("Timed out"), - }) - util.Log.Errorf("Timed out: %s", servername) - } - } - } - } - return -} - -func detectContainerOSesOnServer(containerHost osTypeInterface) (oses []osTypeInterface) { - containerHostInfo := containerHost.getServerInfo() - if len(containerHostInfo.ContainersIncluded) == 0 { - return - } - - running, err := containerHost.runningContainers() - if err != nil { - containerHost.setErrs([]error{xerrors.Errorf( - "Failed to get running containers on %s. err: %w", - containerHost.getServerInfo().ServerName, err)}) - return append(oses, containerHost) - } - - if containerHostInfo.ContainersIncluded[0] == "${running}" { - for _, containerInfo := range running { - found := false - for _, ex := range containerHost.getServerInfo().ContainersExcluded { - if containerInfo.Name == ex || containerInfo.ContainerID == ex { - found = true - } - } - if found { - continue - } - - copied := containerHostInfo - copied.SetContainer(config.Container{ - ContainerID: containerInfo.ContainerID, - Name: containerInfo.Name, - Image: containerInfo.Image, - }) - os := detectOS(copied) - oses = append(oses, os) - } - return oses - } - - exitedContainers, err := containerHost.exitedContainers() - if err != nil { - containerHost.setErrs([]error{xerrors.Errorf( - "Failed to get exited containers on %s. err: %w", - containerHost.getServerInfo().ServerName, err)}) - return append(oses, containerHost) - } - - var exited, unknown []string - for _, container := range containerHostInfo.ContainersIncluded { - found := false - for _, c := range running { - if c.ContainerID == container || c.Name == container { - copied := containerHostInfo - copied.SetContainer(c) - os := detectOS(copied) - oses = append(oses, os) - found = true - break - } - } - - if !found { - foundInExitedContainers := false - for _, c := range exitedContainers { - if c.ContainerID == container || c.Name == container { - exited = append(exited, container) - foundInExitedContainers = true - break - } - } - if !foundInExitedContainers { - unknown = append(unknown, container) - } - } - } - if 0 < len(exited) || 0 < len(unknown) { - containerHost.setErrs([]error{xerrors.Errorf( - "Some containers on %s are exited or unknown. exited: %s, unknown: %s", - containerHost.getServerInfo().ServerName, exited, unknown)}) - return append(oses, containerHost) - } - return oses -} - -// CheckScanModes checks scan mode -func CheckScanModes() error { +// checkScanModes checks scan mode +func (s Scanner) checkScanModes() error { for _, s := range servers { if err := s.checkScanMode(); err != nil { return xerrors.Errorf("servers.%s.scanMode err: %w", @@ -405,25 +498,30 @@ func CheckScanModes() error { return nil } -// CheckDependencies checks dependencies are installed on target servers. -func CheckDependencies(timeoutSec int) { +// checkDependencies checks dependencies are installed on target servers. +func (s Scanner) checkDependencies() { parallelExec(func(o osTypeInterface) error { return o.checkDeps() - }, timeoutSec) + }, s.TimeoutSec) return } -// CheckIfSudoNoPasswd checks whether vuls can sudo with nopassword via SSH -func CheckIfSudoNoPasswd(timeoutSec int) { +// checkIfSudoNoPasswd checks whether vuls can sudo with nopassword via SSH +func (s Scanner) checkIfSudoNoPasswd() { parallelExec(func(o osTypeInterface) error { return o.checkIfSudoNoPasswd() - }, timeoutSec) + }, s.TimeoutSec) return } -// DetectPlatforms detects the platform of each servers. -func DetectPlatforms(timeoutSec int) { - detectPlatforms(timeoutSec) +// detectPlatform detects the platform of each servers. +func (s Scanner) detectPlatform() { + parallelExec(func(o osTypeInterface) error { + o.detectPlatform() + // Logging only if platform can not be specified + return nil + }, s.TimeoutSec) + for i, s := range servers { if s.getServerInfo().IsContainer() { util.Log.Infof("(%d/%d) %s on %s is running on %s", @@ -444,18 +542,14 @@ func DetectPlatforms(timeoutSec int) { return } -func detectPlatforms(timeoutSec int) { +// detectIPS detects the IPS of each servers. +func (s Scanner) detectIPS() { parallelExec(func(o osTypeInterface) error { - o.detectPlatform() - // Logging only if platform can not be specified + o.detectIPS() + // Logging only if IPS can not be specified return nil - }, timeoutSec) - return -} + }, s.TimeoutSec) -// DetectIPSs detects the IPS of each servers. -func DetectIPSs(timeoutSec int) { - detectIPSs(timeoutSec) for i, s := range servers { if !s.getServerInfo().IsContainer() { util.Log.Infof("(%d/%d) %s has %d IPS integration", @@ -467,21 +561,13 @@ func DetectIPSs(timeoutSec int) { } } -func detectIPSs(timeoutSec int) { - parallelExec(func(o osTypeInterface) error { - o.detectIPSs() - // Logging only if IPS can not be specified - return nil - }, timeoutSec) -} - -// Scan scan -func Scan(timeoutSec int) error { +// execScan scan +func (s Scanner) execScan() error { if len(servers) == 0 { return xerrors.New("No server defined. Check the configuration") } - if err := setupChangelogCache(); err != nil { + if err := s.setupChangelogCache(); err != nil { return err } defer func() { @@ -496,117 +582,28 @@ func Scan(timeoutSec int) error { return err } - results, err := GetScanResults(scannedAt, timeoutSec) + results, err := s.getScanResults(scannedAt) if err != nil { return err } for i, r := range results { - if s, ok := config.Conf.Servers[r.ServerName]; ok { - results[i] = r.ClearFields(s.IgnoredJSONKeys) + if server, ok := s.Targets[r.ServerName]; ok { + results[i] = r.ClearFields(server.IgnoredJSONKeys) } } return writeScanResults(dir, results) } -// ViaHTTP scans servers by HTTP header and body -func ViaHTTP(header http.Header, body string) (models.ScanResult, error) { - family := header.Get("X-Vuls-OS-Family") - if family == "" { - return models.ScanResult{}, errOSFamilyHeader - } - - release := header.Get("X-Vuls-OS-Release") - if release == "" { - return models.ScanResult{}, errOSReleaseHeader - } - - kernelRelease := header.Get("X-Vuls-Kernel-Release") - if kernelRelease == "" { - util.Log.Warn("If X-Vuls-Kernel-Release is not specified, there is a possibility of false detection") - } - - kernelVersion := header.Get("X-Vuls-Kernel-Version") - if family == config.Debian && kernelVersion == "" { - return models.ScanResult{}, errKernelVersionHeader - } - - serverName := header.Get("X-Vuls-Server-Name") - if config.Conf.ToLocalFile && serverName == "" { - return models.ScanResult{}, errServerNameHeader - } - - distro := config.Distro{ - Family: family, - Release: release, - } - - kernel := models.Kernel{ - Release: kernelRelease, - Version: kernelVersion, - } - base := base{ - Distro: distro, - osPackages: osPackages{ - Kernel: kernel, - }, - log: util.Log, - } - - var osType osTypeInterface - switch family { - case config.Debian, config.Ubuntu: - osType = &debian{base: base} - case config.RedHat: - osType = &rhel{ - redhatBase: redhatBase{base: base}, - } - case config.CentOS: - osType = ¢os{ - redhatBase: redhatBase{base: base}, - } - case config.Oracle: - osType = &oracle{ - redhatBase: redhatBase{base: base}, - } - case config.Amazon: - osType = &amazon{ - redhatBase: redhatBase{base: base}, - } - default: - return models.ScanResult{}, xerrors.Errorf("Server mode for %s is not implemented yet", family) - } - - installedPackages, srcPackages, err := osType.parseInstalledPackages(body) - if err != nil { - return models.ScanResult{}, err - } - - result := models.ScanResult{ - ServerName: serverName, - Family: family, - Release: release, - RunningKernel: models.Kernel{ - Release: kernelRelease, - Version: kernelVersion, - }, - Packages: installedPackages, - SrcPackages: srcPackages, - ScannedCves: models.VulnInfos{}, - } - - return result, nil -} - -func setupChangelogCache() error { +func (s Scanner) setupChangelogCache() error { needToSetupCache := false for _, s := range servers { switch s.getDistro().Family { - case config.Raspbian: + case constant.Raspbian: needToSetupCache = true break - case config.Ubuntu, config.Debian: + case constant.Ubuntu, constant.Debian: //TODO changelog cache for RedHat, Oracle, Amazon, CentOS is not implemented yet. if s.getServerInfo().Mode.IsDeep() { needToSetupCache = true @@ -615,15 +612,15 @@ func setupChangelogCache() error { } } if needToSetupCache { - if err := cache.SetupBolt(config.Conf.CacheDBPath, util.Log); err != nil { + if err := cache.SetupBolt(s.CacheDBPath, util.Log); err != nil { return err } } return nil } -// GetScanResults returns ScanResults from -func GetScanResults(scannedAt time.Time, timeoutSec int) (results models.ScanResults, err error) { +// getScanResults returns ScanResults +func (s Scanner) getScanResults(scannedAt time.Time) (results models.ScanResults, err error) { parallelExec(func(o osTypeInterface) (err error) { if o.getServerInfo().Module.IsScanOSPkg() { if err = o.preCure(); err != nil { @@ -652,7 +649,7 @@ func GetScanResults(scannedAt time.Time, timeoutSec int) (results models.ScanRes } } return nil - }, timeoutSec) + }, s.ScanTimeoutSec) hostname, _ := os.Hostname() ipv4s, ipv6s, err := util.IP() @@ -662,7 +659,7 @@ func GetScanResults(scannedAt time.Time, timeoutSec int) (results models.ScanRes for _, s := range append(servers, errServers...) { r := s.convertToModel() - checkEOL(&r) + r.CheckEOL() r.ScannedAt = scannedAt r.ScannedVersion = config.Version r.ScannedRevision = config.Revision @@ -679,93 +676,3 @@ func GetScanResults(scannedAt time.Time, timeoutSec int) (results models.ScanRes } return results, nil } - -func checkEOL(r *models.ScanResult) { - switch r.Family { - case config.ServerTypePseudo, config.Raspbian: - return - } - - eol, found := config.GetEOL(r.Family, r.Release) - if !found { - r.Warnings = append(r.Warnings, - fmt.Sprintf("Failed to check EOL. Register the issue to https://github.com/future-architect/vuls/issues with the information in `Family: %s Release: %s`", - r.Family, r.Release)) - return - } - - now := time.Now() - if eol.IsStandardSupportEnded(now) { - r.Warnings = append(r.Warnings, "Standard OS support is EOL(End-of-Life). Purchase extended support if available or Upgrading your OS is strongly recommended.") - if eol.ExtendedSupportUntil.IsZero() { - return - } - if !eol.IsExtendedSuppportEnded(now) { - r.Warnings = append(r.Warnings, - fmt.Sprintf("Extended support available until %s. Check the vendor site.", - eol.ExtendedSupportUntil.Format("2006-01-02"))) - } else { - r.Warnings = append(r.Warnings, - "Extended support is also EOL. There are many Vulnerabilities that are not detected, Upgrading your OS strongly recommended.") - } - } else if !eol.StandardSupportUntil.IsZero() && - now.AddDate(0, 3, 0).After(eol.StandardSupportUntil) { - r.Warnings = append(r.Warnings, - fmt.Sprintf("Standard OS support will be end in 3 months. EOL date: %s", - eol.StandardSupportUntil.Format("2006-01-02"))) - } -} - -func writeScanResults(jsonDir string, results models.ScanResults) error { - config.Conf.FormatJSON = true - ws := []report.ResultWriter{ - report.LocalFileWriter{CurrentDir: jsonDir}, - } - for _, w := range ws { - if err := w.Write(results...); err != nil { - return xerrors.Errorf("Failed to write summary: %s", err) - } - } - - report.StdoutWriter{}.WriteScanSummary(results...) - - errServerNames := []string{} - for _, r := range results { - if 0 < len(r.Errors) { - errServerNames = append(errServerNames, r.ServerName) - } - } - if 0 < len(errServerNames) { - return fmt.Errorf("An error occurred on %s", errServerNames) - } - return nil -} - -// EnsureResultDir ensures the directory for scan results -func EnsureResultDir(scannedAt time.Time) (currentDir string, err error) { - jsonDirName := scannedAt.Format(time.RFC3339) - - resultsDir := config.Conf.ResultsDir - if len(resultsDir) == 0 { - wd, _ := os.Getwd() - resultsDir = filepath.Join(wd, "results") - } - jsonDir := filepath.Join(resultsDir, jsonDirName) - if err := os.MkdirAll(jsonDir, 0700); err != nil { - return "", xerrors.Errorf("Failed to create dir: %w", err) - } - - symlinkPath := filepath.Join(resultsDir, "current") - if _, err := os.Lstat(symlinkPath); err == nil { - if err := os.Remove(symlinkPath); err != nil { - return "", xerrors.Errorf( - "Failed to remove symlink. path: %s, err: %w", symlinkPath, err) - } - } - - if err := os.Symlink(jsonDir, symlinkPath); err != nil { - return "", xerrors.Errorf( - "Failed to create symlink: path: %s, err: %w", symlinkPath, err) - } - return jsonDir, nil -} diff --git a/scan/serverapi_test.go b/scanner/serverapi_test.go similarity index 95% rename from scan/serverapi_test.go rename to scanner/serverapi_test.go index b2e35075..672d2c35 100644 --- a/scan/serverapi_test.go +++ b/scanner/serverapi_test.go @@ -1,16 +1,17 @@ -package scan +package scanner import ( "net/http" "testing" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" ) func TestViaHTTP(t *testing.T) { r := newRHEL(config.ServerInfo{}) - r.Distro = config.Distro{Family: config.RedHat} + r.Distro = config.Distro{Family: constant.RedHat} var tests = []struct { header map[string]string @@ -102,7 +103,7 @@ func TestViaHTTP(t *testing.T) { header.Set(k, v) } - result, err := ViaHTTP(header, tt.body) + result, err := ViaHTTP(header, tt.body, false) if err != tt.wantErr { t.Errorf("error: expected %s, actual: %s", tt.wantErr, err) } diff --git a/scan/suse.go b/scanner/suse.go similarity index 95% rename from scan/suse.go rename to scanner/suse.go index 5fee8103..816f526d 100644 --- a/scan/suse.go +++ b/scanner/suse.go @@ -1,4 +1,4 @@ -package scan +package scanner import ( "bufio" @@ -7,6 +7,7 @@ import ( "strings" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/future-architect/vuls/util" "golang.org/x/xerrors" @@ -53,7 +54,7 @@ func detectSUSE(c config.ServerInfo) (bool, osTypeInterface) { if len(result) == 2 { //TODO check opensuse or opensuse.leap s := newSUSE(c) - s.setDistro(config.OpenSUSE, result[1]) + s.setDistro(constant.OpenSUSE, result[1]) return true, s } @@ -65,7 +66,7 @@ func detectSUSE(c config.ServerInfo) (bool, osTypeInterface) { result = re.FindStringSubmatch(strings.TrimSpace(r.Stdout)) if len(result) == 2 { s := newSUSE(c) - s.setDistro(config.SUSEEnterpriseServer, + s.setDistro(constant.SUSEEnterpriseServer, fmt.Sprintf("%s.%s", version, result[1])) return true, s } @@ -82,11 +83,11 @@ func detectSUSE(c config.ServerInfo) (bool, osTypeInterface) { func (o *suse) parseOSRelease(content string) (name string, ver string) { if strings.Contains(content, "ID=opensuse") { //TODO check opensuse or opensuse.leap - name = config.OpenSUSE + name = constant.OpenSUSE } else if strings.Contains(content, `NAME="SLES"`) { - name = config.SUSEEnterpriseServer + name = constant.SUSEEnterpriseServer } else if strings.Contains(content, `NAME="SLES_SAP"`) { - name = config.SUSEEnterpriseServer + name = constant.SUSEEnterpriseServer } else { util.Log.Warnf("Failed to parse SUSE edition: %s", content) return "unknown", "unknown" diff --git a/scan/suse_test.go b/scanner/suse_test.go similarity index 95% rename from scan/suse_test.go rename to scanner/suse_test.go index aff1169f..cbdb5dbe 100644 --- a/scan/suse_test.go +++ b/scanner/suse_test.go @@ -1,10 +1,11 @@ -package scan +package scanner import ( "reflect" "testing" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" "github.com/k0kubun/pp" ) @@ -107,7 +108,7 @@ func TestParseOSRelease(t *testing.T) { in: `NAME="openSUSE Leap" ID=opensuse VERSION_ID="42.3.4"`, - name: config.OpenSUSE, + name: constant.OpenSUSE, ver: "42.3.4", }, { @@ -115,7 +116,7 @@ VERSION_ID="42.3.4"`, VERSION="12-SP1" VERSION_ID="12.1" ID="sles"`, - name: config.SUSEEnterpriseServer, + name: constant.SUSEEnterpriseServer, ver: "12.1", }, { @@ -123,7 +124,7 @@ ID="sles"`, VERSION="12-SP1" VERSION_ID="12.1.0.1" ID="sles"`, - name: config.SUSEEnterpriseServer, + name: constant.SUSEEnterpriseServer, ver: "12.1.0.1", }, } diff --git a/scan/unknownDistro.go b/scanner/unknownDistro.go similarity index 97% rename from scan/unknownDistro.go rename to scanner/unknownDistro.go index 10ffdcab..bb6563d3 100644 --- a/scan/unknownDistro.go +++ b/scanner/unknownDistro.go @@ -1,4 +1,4 @@ -package scan +package scanner import "github.com/future-architect/vuls/models" diff --git a/scanner/utils.go b/scanner/utils.go new file mode 100644 index 00000000..91512f15 --- /dev/null +++ b/scanner/utils.go @@ -0,0 +1,96 @@ +package scanner + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" + "github.com/future-architect/vuls/models" + "github.com/future-architect/vuls/reporter" + "github.com/future-architect/vuls/util" + "golang.org/x/xerrors" +) + +func isRunningKernel(pack models.Package, family string, kernel models.Kernel) (isKernel, running bool) { + switch family { + case constant.SUSEEnterpriseServer: + if pack.Name == "kernel-default" { + // Remove the last period and later because uname don't show that. + ss := strings.Split(pack.Release, ".") + rel := strings.Join(ss[0:len(ss)-1], ".") + ver := fmt.Sprintf("%s-%s-default", pack.Version, rel) + return true, kernel.Release == ver + } + return false, false + + case constant.RedHat, constant.Oracle, constant.CentOS, constant.Amazon: + switch pack.Name { + case "kernel", "kernel-devel": + ver := fmt.Sprintf("%s-%s.%s", pack.Version, pack.Release, pack.Arch) + return true, kernel.Release == ver + } + return false, false + + default: + util.Log.Warnf("Reboot required is not implemented yet: %s, %v", family, kernel) + } + return false, false +} + +// EnsureResultDir ensures the directory for scan results +func EnsureResultDir(scannedAt time.Time) (currentDir string, err error) { + jsonDirName := scannedAt.Format(time.RFC3339) + + resultsDir := config.Conf.ResultsDir + if len(resultsDir) == 0 { + wd, _ := os.Getwd() + resultsDir = filepath.Join(wd, "results") + } + jsonDir := filepath.Join(resultsDir, jsonDirName) + if err := os.MkdirAll(jsonDir, 0700); err != nil { + return "", xerrors.Errorf("Failed to create dir: %w", err) + } + + symlinkPath := filepath.Join(resultsDir, "current") + if _, err := os.Lstat(symlinkPath); err == nil { + if err := os.Remove(symlinkPath); err != nil { + return "", xerrors.Errorf( + "Failed to remove symlink. path: %s, err: %w", symlinkPath, err) + } + } + + if err := os.Symlink(jsonDir, symlinkPath); err != nil { + return "", xerrors.Errorf( + "Failed to create symlink: path: %s, err: %w", symlinkPath, err) + } + return jsonDir, nil +} + +func writeScanResults(jsonDir string, results models.ScanResults) error { + ws := []reporter.ResultWriter{reporter.LocalFileWriter{ + CurrentDir: jsonDir, + FormatJSON: true, + }} + for _, w := range ws { + if err := w.Write(results...); err != nil { + return xerrors.Errorf("Failed to write summary: %s", err) + } + } + + reporter.StdoutWriter{}.WriteScanSummary(results...) + + errServerNames := []string{} + for _, r := range results { + if 0 < len(r.Errors) { + errServerNames = append(errServerNames, r.ServerName) + } + } + if 0 < len(errServerNames) { + return fmt.Errorf("An error occurred on %s", errServerNames) + } + return nil +} diff --git a/scan/utils_test.go b/scanner/utils_test.go similarity index 84% rename from scan/utils_test.go rename to scanner/utils_test.go index a8e867bd..454106fb 100644 --- a/scan/utils_test.go +++ b/scanner/utils_test.go @@ -1,15 +1,16 @@ -package scan +package scanner import ( "testing" "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/models" ) func TestIsRunningKernelSUSE(t *testing.T) { r := newSUSE(config.ServerInfo{}) - r.Distro = config.Distro{Family: config.SUSEEnterpriseServer} + r.Distro = config.Distro{Family: constant.SUSEEnterpriseServer} kernel := models.Kernel{ Release: "4.4.74-92.35-default", @@ -29,7 +30,7 @@ func TestIsRunningKernelSUSE(t *testing.T) { Release: "92.35.1", Arch: "x86_64", }, - family: config.SUSEEnterpriseServer, + family: constant.SUSEEnterpriseServer, kernel: kernel, expected: true, }, @@ -40,7 +41,7 @@ func TestIsRunningKernelSUSE(t *testing.T) { Release: "92.20.2", Arch: "x86_64", }, - family: config.SUSEEnterpriseServer, + family: constant.SUSEEnterpriseServer, kernel: kernel, expected: false, }, @@ -56,7 +57,7 @@ func TestIsRunningKernelSUSE(t *testing.T) { func TestIsRunningKernelRedHatLikeLinux(t *testing.T) { r := newAmazon(config.ServerInfo{}) - r.Distro = config.Distro{Family: config.Amazon} + r.Distro = config.Distro{Family: constant.Amazon} kernel := models.Kernel{ Release: "4.9.43-17.38.amzn1.x86_64", @@ -76,7 +77,7 @@ func TestIsRunningKernelRedHatLikeLinux(t *testing.T) { Release: "17.38.amzn1", Arch: "x86_64", }, - family: config.Amazon, + family: constant.Amazon, kernel: kernel, expected: true, }, @@ -87,7 +88,7 @@ func TestIsRunningKernelRedHatLikeLinux(t *testing.T) { Release: "16.35.amzn1", Arch: "x86_64", }, - family: config.Amazon, + family: constant.Amazon, kernel: kernel, expected: false, }, diff --git a/server/empty.go b/server/empty.go deleted file mode 100644 index abb4e431..00000000 --- a/server/empty.go +++ /dev/null @@ -1 +0,0 @@ -package server diff --git a/server/server.go b/server/server.go index 3f1e35ef..34115fb5 100644 --- a/server/server.go +++ b/server/server.go @@ -11,16 +11,17 @@ import ( "net/http" "time" - c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/detector" "github.com/future-architect/vuls/models" - "github.com/future-architect/vuls/report" - "github.com/future-architect/vuls/scan" + "github.com/future-architect/vuls/reporter" + "github.com/future-architect/vuls/scanner" "github.com/future-architect/vuls/util" ) // VulsHandler is used for vuls server mode type VulsHandler struct { - DBclient report.DBClient + DBclient detector.DBClient + ToLocalFile bool } // ServeHTTP is http handler @@ -48,7 +49,7 @@ func (h VulsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - if result, err = scan.ViaHTTP(req.Header, buf.String()); err != nil { + if result, err = scanner.ViaHTTP(req.Header, buf.String(), h.ToLocalFile); err != nil { util.Log.Error(err) http.Error(w, err.Error(), http.StatusBadRequest) return @@ -59,13 +60,13 @@ func (h VulsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { return } - if err := report.DetectPkgCves(h.DBclient, &result); err != nil { + if err := detector.DetectPkgCves(h.DBclient, &result); err != nil { util.Log.Errorf("Failed to detect Pkg CVE: %+v", err) http.Error(w, err.Error(), http.StatusServiceUnavailable) return } - if err := report.FillCveInfo(h.DBclient, &result); err != nil { + if err := detector.FillCveInfo(h.DBclient, &result); err != nil { util.Log.Errorf("Failed to fill CVE detailed info: %+v", err) http.Error(w, err.Error(), http.StatusServiceUnavailable) return @@ -78,24 +79,26 @@ func (h VulsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } // report - reports := []report.ResultWriter{ - report.HTTPResponseWriter{Writer: w}, + reports := []reporter.ResultWriter{ + reporter.HTTPResponseWriter{Writer: w}, } - if c.Conf.ToLocalFile { + if h.ToLocalFile { scannedAt := result.ScannedAt if scannedAt.IsZero() { scannedAt = time.Now().Truncate(1 * time.Hour) result.ScannedAt = scannedAt } - dir, err := scan.EnsureResultDir(scannedAt) + dir, err := scanner.EnsureResultDir(scannedAt) if err != nil { util.Log.Errorf("Failed to ensure the result directory: %+v", err) http.Error(w, err.Error(), http.StatusServiceUnavailable) return } - reports = append(reports, report.LocalFileWriter{ + // sever subcmd doesn't have diff option + reports = append(reports, reporter.LocalFileWriter{ CurrentDir: dir, + FormatJSON: true, }) } diff --git a/subcmds/configtest.go b/subcmds/configtest.go index f7db1558..1064a3f1 100644 --- a/subcmds/configtest.go +++ b/subcmds/configtest.go @@ -11,7 +11,7 @@ import ( "github.com/google/subcommands" c "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/scan" + "github.com/future-architect/vuls/scanner" "github.com/future-architect/vuls/util" ) @@ -64,9 +64,6 @@ func (p *ConfigtestCmd) SetFlags(f *flag.FlagSet) { f.StringVar(&c.Conf.HTTPProxy, "http-proxy", "", "http://proxy-url:port (default: empty)") - f.BoolVar(&c.Conf.SSHNative, "ssh-native-insecure", false, - "Use Native Go implementation of SSH. Default: Use the external command") - f.BoolVar(&c.Conf.Vvv, "vvv", false, "ssh -vvv") } @@ -105,12 +102,12 @@ func (p *ConfigtestCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfa servernames = f.Args() } - target := make(map[string]c.ServerInfo) + targets := make(map[string]c.ServerInfo) for _, arg := range servernames { found := false for servername, info := range c.Conf.Servers { if servername == arg { - target[servername] = info + targets[servername] = info found = true break } @@ -121,7 +118,7 @@ func (p *ConfigtestCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfa } } if 0 < len(servernames) { - c.Conf.Servers = target + c.Conf.Servers = targets } util.Log.Info("Validating config...") @@ -129,28 +126,15 @@ func (p *ConfigtestCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfa return subcommands.ExitUsageError } - util.Log.Info("Detecting Server/Container OS... ") - if err := scan.InitServers(p.timeoutSec); err != nil { - util.Log.Errorf("Failed to init servers. err: %+v", err) + s := scanner.Scanner{ + TimeoutSec: p.timeoutSec, + Targets: targets, + } + + if err := s.Configtest(); err != nil { + util.Log.Errorf("Failed to configtest: %+v", err) return subcommands.ExitFailure } - util.Log.Info("Checking Scan Modes...") - if err := scan.CheckScanModes(); err != nil { - util.Log.Errorf("Fix config.toml. err: %+v", err) - return subcommands.ExitFailure - } - - util.Log.Info("Checking dependencies...") - scan.CheckDependencies(p.timeoutSec) - - util.Log.Info("Checking sudo settings...") - scan.CheckIfSudoNoPasswd(p.timeoutSec) - - util.Log.Info("It can be scanned with fast scan mode even if warn or err messages are displayed due to lack of dependent packages or sudo settings in fast-root or deep scan mode") - - if scan.PrintSSHableServerNames() { - return subcommands.ExitSuccess - } - return subcommands.ExitFailure + return subcommands.ExitSuccess } diff --git a/subcmds/history.go b/subcmds/history.go index 616aca04..5152ccbe 100644 --- a/subcmds/history.go +++ b/subcmds/history.go @@ -10,7 +10,7 @@ import ( "strings" c "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/report" + "github.com/future-architect/vuls/reporter" "github.com/google/subcommands" ) @@ -45,7 +45,7 @@ func (p *HistoryCmd) SetFlags(f *flag.FlagSet) { // Execute execute func (p *HistoryCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { - dirs, err := report.ListValidJSONDirs() + dirs, err := reporter.ListValidJSONDirs() if err != nil { return subcommands.ExitFailure } diff --git a/subcmds/report.go b/subcmds/report.go index 973cb0d1..4b505fcc 100644 --- a/subcmds/report.go +++ b/subcmds/report.go @@ -11,8 +11,9 @@ import ( "github.com/aquasecurity/trivy/pkg/utils" "github.com/future-architect/vuls/config" c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/detector" "github.com/future-architect/vuls/models" - "github.com/future-architect/vuls/report" + "github.com/future-architect/vuls/reporter" "github.com/future-architect/vuls/util" "github.com/google/subcommands" "github.com/k0kubun/pp" @@ -21,7 +22,26 @@ import ( // ReportCmd is subcommand for reporting type ReportCmd struct { configPath string - httpConf c.HTTPConf + + formatJSON bool + formatOneEMail bool + formatCsv bool + formatFullText bool + formatOneLineText bool + formatList bool + gzip bool + + toSlack bool + toChatWork bool + toTelegram bool + toEmail bool + toSyslog bool + toLocalFile bool + toS3 bool + toAzureBlob bool + toHTTP bool + + toHTTPURL string } // Name return subcommand name @@ -119,32 +139,31 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) { &c.Conf.HTTPProxy, "http-proxy", "", "http://proxy-url:port (default: empty)") - f.BoolVar(&c.Conf.FormatJSON, "format-json", false, "JSON format") - f.BoolVar(&c.Conf.FormatCsvList, "format-csv", false, "CSV format") - f.BoolVar(&c.Conf.FormatOneEMail, "format-one-email", false, + f.BoolVar(&p.formatJSON, "format-json", false, "JSON format") + f.BoolVar(&p.formatCsv, "format-csv", false, "CSV format") + f.BoolVar(&p.formatOneEMail, "format-one-email", false, "Send all the host report via only one EMail (Specify with -to-email)") - f.BoolVar(&c.Conf.FormatOneLineText, "format-one-line-text", false, + f.BoolVar(&p.formatOneLineText, "format-one-line-text", false, "One line summary in plain text") - f.BoolVar(&c.Conf.FormatList, "format-list", false, "Display as list format") - f.BoolVar(&c.Conf.FormatFullText, "format-full-text", false, + f.BoolVar(&p.formatList, "format-list", false, "Display as list format") + f.BoolVar(&p.formatFullText, "format-full-text", false, "Detail report in plain text") - f.BoolVar(&c.Conf.ToSlack, "to-slack", false, "Send report via Slack") - f.BoolVar(&c.Conf.ToChatWork, "to-chatwork", false, "Send report via chatwork") - f.BoolVar(&c.Conf.ToTelegram, "to-telegram", false, "Send report via Telegram") - f.BoolVar(&c.Conf.ToEmail, "to-email", false, "Send report via Email") - f.BoolVar(&c.Conf.ToSyslog, "to-syslog", false, "Send report via Syslog") - f.BoolVar(&c.Conf.ToLocalFile, "to-localfile", false, "Write report to localfile") - f.BoolVar(&c.Conf.ToS3, "to-s3", false, - "Write report to S3 (bucket/yyyyMMdd_HHmm/servername.json/txt)") - f.BoolVar(&c.Conf.ToHTTP, "to-http", false, "Send report via HTTP POST") - f.BoolVar(&c.Conf.ToAzureBlob, "to-azure-blob", false, + f.BoolVar(&p.toSlack, "to-slack", false, "Send report via Slack") + f.BoolVar(&p.toChatWork, "to-chatwork", false, "Send report via chatwork") + f.BoolVar(&p.toTelegram, "to-telegram", false, "Send report via Telegram") + f.BoolVar(&p.toEmail, "to-email", false, "Send report via Email") + f.BoolVar(&p.toSyslog, "to-syslog", false, "Send report via Syslog") + f.BoolVar(&p.toLocalFile, "to-localfile", false, "Write report to localfile") + f.BoolVar(&p.toS3, "to-s3", false, "Write report to S3 (bucket/yyyyMMdd_HHmm/servername.json/txt)") + f.BoolVar(&p.toHTTP, "to-http", false, "Send report via HTTP POST") + f.BoolVar(&p.toAzureBlob, "to-azure-blob", false, "Write report to Azure Storage blob (container/yyyyMMdd_HHmm/servername.json/txt)") - f.BoolVar(&c.Conf.GZIP, "gzip", false, "gzip compression") + f.BoolVar(&p.gzip, "gzip", false, "gzip compression") f.BoolVar(&c.Conf.Pipe, "pipe", false, "Use args passed via PIPE") - f.StringVar(&p.httpConf.URL, "http", "", "-to-http http://vuls-report") + f.StringVar(&p.toHTTPURL, "http", "", "-to-http http://vuls-report") f.StringVar(&c.Conf.TrivyCacheDBDir, "trivy-cachedb-dir", utils.DefaultCacheDir(), "/path/to/dir") @@ -153,23 +172,33 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) { // Execute execute func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { util.Log = util.NewCustomLogger(c.ServerInfo{}) + if err := c.Load(p.configPath, ""); err != nil { util.Log.Errorf("Error loading %s, %+v", p.configPath, err) return subcommands.ExitUsageError } - c.Conf.HTTP.Init(p.httpConf) + c.Conf.Slack.Enabled = p.toSlack + c.Conf.ChatWork.Enabled = p.toChatWork + c.Conf.Telegram.Enabled = p.toTelegram + c.Conf.EMail.Enabled = p.toEmail + c.Conf.Syslog.Enabled = p.toSyslog + c.Conf.AWS.Enabled = p.toS3 + c.Conf.Azure.Enabled = p.toAzureBlob + c.Conf.HTTP.Enabled = p.toHTTP + + //TODO refactor + c.Conf.HTTP.Init(p.toHTTPURL) if c.Conf.Diff { - c.Conf.DiffPlus = true - c.Conf.DiffMinus = true + c.Conf.DiffPlus, c.Conf.DiffMinus = true, true } var dir string var err error if c.Conf.DiffPlus || c.Conf.DiffMinus { - dir, err = report.JSONDir([]string{}) + dir, err = reporter.JSONDir([]string{}) } else { - dir, err = report.JSONDir(f.Args()) + dir, err = reporter.JSONDir(f.Args()) } if err != nil { util.Log.Errorf("Failed to read from JSON: %+v", err) @@ -181,13 +210,13 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} return subcommands.ExitUsageError } - if !(c.Conf.FormatJSON || c.Conf.FormatOneLineText || - c.Conf.FormatList || c.Conf.FormatFullText || c.Conf.FormatCsvList) { - c.Conf.FormatList = true + if !(p.formatJSON || p.formatOneLineText || + p.formatList || p.formatFullText || p.formatCsv) { + p.formatList = true } var loaded models.ScanResults - if loaded, err = report.LoadScanResults(dir); err != nil { + if loaded, err = reporter.LoadScanResults(dir); err != nil { util.Log.Error(err) return subcommands.ExitFailure } @@ -211,8 +240,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} for _, r := range res { util.Log.Debugf("%s: %s", - r.ServerInfo(), - pp.Sprintf("%s", c.Conf.Servers[r.ServerName])) + r.ServerInfo(), pp.Sprintf("%s", c.Conf.Servers[r.ServerName])) } util.Log.Info("Validating db config...") @@ -233,7 +261,8 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } } - dbclient, locked, err := report.NewDBClient(report.DBClientConf{ + // TODO move into fillcveInfos + dbclient, locked, err := detector.NewDBClient(detector.DBClientConf{ CveDictCnf: c.Conf.CveDict, OvalDictCnf: c.Conf.OvalDict, GostCnf: c.Conf.Gost, @@ -251,74 +280,107 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } defer dbclient.CloseDB() - if res, err = report.FillCveInfos(*dbclient, res, dir); err != nil { + // TODO pass conf by arg + if res, err = detector.Detect(*dbclient, res, dir); err != nil { util.Log.Errorf("%+v", err) return subcommands.ExitFailure } // report - reports := []report.ResultWriter{ - report.StdoutWriter{}, + reports := []reporter.ResultWriter{ + reporter.StdoutWriter{ + FormatCsv: p.formatCsv, + FormatFullText: p.formatFullText, + FormatOneLineText: p.formatOneLineText, + FormatList: p.formatList, + }, } - if c.Conf.ToSlack { - reports = append(reports, report.SlackWriter{}) - } - - if c.Conf.ToChatWork { - reports = append(reports, report.ChatWorkWriter{}) - } - - if c.Conf.ToTelegram { - reports = append(reports, report.TelegramWriter{}) - } - - if c.Conf.ToEmail { - reports = append(reports, report.EMailWriter{}) - } - - if c.Conf.ToSyslog { - reports = append(reports, report.SyslogWriter{}) - } - - if c.Conf.ToHTTP { - reports = append(reports, report.HTTPRequestWriter{}) - } - - if c.Conf.ToLocalFile { - reports = append(reports, report.LocalFileWriter{ - CurrentDir: dir, + if p.toSlack { + reports = append(reports, reporter.SlackWriter{ + FormatOneLineText: p.formatOneLineText, }) } - if c.Conf.ToS3 { - if err := report.CheckIfBucketExists(); err != nil { + if p.toChatWork { + reports = append(reports, reporter.ChatWorkWriter{}) + } + + if p.toTelegram { + reports = append(reports, reporter.TelegramWriter{}) + } + + if p.toEmail { + reports = append(reports, reporter.EMailWriter{ + FormatOneEMail: p.formatOneEMail, + FormatOneLineText: p.formatOneLineText, + FormatList: p.formatList, + }) + } + + if p.toSyslog { + reports = append(reports, reporter.SyslogWriter{}) + } + + if p.toHTTP { + reports = append(reports, reporter.HTTPRequestWriter{}) + } + + if p.toLocalFile { + reports = append(reports, reporter.LocalFileWriter{ + CurrentDir: dir, + DiffPlus: c.Conf.DiffPlus, + DiffMinus: c.Conf.DiffMinus, + FormatJSON: p.formatJSON, + FormatCsv: p.formatCsv, + FormatFullText: p.formatFullText, + FormatOneLineText: p.formatOneLineText, + FormatList: p.formatList, + Gzip: p.gzip, + }) + } + + if p.toS3 { + if err := reporter.CheckIfBucketExists(); err != nil { util.Log.Errorf("Check if there is a bucket beforehand: %s, err: %+v", c.Conf.AWS.S3Bucket, err) return subcommands.ExitUsageError } - reports = append(reports, report.S3Writer{}) + reports = append(reports, reporter.S3Writer{ + FormatJSON: p.formatJSON, + FormatFullText: p.formatFullText, + FormatOneLineText: p.formatOneLineText, + FormatList: p.formatList, + Gzip: p.gzip, + }) } - if c.Conf.ToAzureBlob { - if len(c.Conf.Azure.AccountName) == 0 { + if p.toAzureBlob { + // TODO refactor + if c.Conf.Azure.AccountName == "" { c.Conf.Azure.AccountName = os.Getenv("AZURE_STORAGE_ACCOUNT") } - if len(c.Conf.Azure.AccountKey) == 0 { + if c.Conf.Azure.AccountKey == "" { c.Conf.Azure.AccountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY") } - if len(c.Conf.Azure.ContainerName) == 0 { + if c.Conf.Azure.ContainerName == "" { util.Log.Error("Azure storage container name is required with -azure-container option") return subcommands.ExitUsageError } - if err := report.CheckIfAzureContainerExists(); err != nil { + if err := reporter.CheckIfAzureContainerExists(); err != nil { util.Log.Errorf("Check if there is a container beforehand: %s, err: %+v", c.Conf.Azure.ContainerName, err) return subcommands.ExitUsageError } - reports = append(reports, report.AzureBlobWriter{}) + reports = append(reports, reporter.AzureBlobWriter{ + FormatJSON: p.formatJSON, + FormatFullText: p.formatFullText, + FormatOneLineText: p.formatOneLineText, + FormatList: p.formatList, + Gzip: p.gzip, + }) } for _, w := range reports { diff --git a/subcmds/saas.go b/subcmds/saas.go index c334efc5..2139b0c4 100644 --- a/subcmds/saas.go +++ b/subcmds/saas.go @@ -8,7 +8,7 @@ import ( c "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/models" - "github.com/future-architect/vuls/report" + "github.com/future-architect/vuls/reporter" "github.com/future-architect/vuls/saas" "github.com/future-architect/vuls/util" "github.com/google/subcommands" @@ -35,19 +35,14 @@ func (*SaaSCmd) Usage() string { [-log-dir=/path/to/log] [-http-proxy=http://192.168.0.1:8080] [-debug] - [-debug-sql] [-quiet] - [-no-progress] ` } // SetFlags set flag func (p *SaaSCmd) SetFlags(f *flag.FlagSet) { - f.StringVar(&c.Conf.Lang, "lang", "en", "[en|ja]") f.BoolVar(&c.Conf.Debug, "debug", false, "debug mode") - f.BoolVar(&c.Conf.DebugSQL, "debug-sql", false, "SQL debug mode") f.BoolVar(&c.Conf.Quiet, "quiet", false, "Quiet mode. No output on stdout") - f.BoolVar(&c.Conf.NoProgress, "no-progress", false, "Suppress progress bar") wd, _ := os.Getwd() defaultConfPath := filepath.Join(wd, "config.toml") @@ -72,7 +67,7 @@ func (p *SaaSCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitUsageError } - dir, err := report.JSONDir(f.Args()) + dir, err := reporter.JSONDir(f.Args()) if err != nil { util.Log.Errorf("Failed to read from JSON: %+v", err) return subcommands.ExitFailure @@ -84,7 +79,7 @@ func (p *SaaSCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } var loaded models.ScanResults - if loaded, err = report.LoadScanResults(dir); err != nil { + if loaded, err = reporter.LoadScanResults(dir); err != nil { util.Log.Error(err) return subcommands.ExitFailure } @@ -108,8 +103,7 @@ func (p *SaaSCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) for _, r := range res { util.Log.Debugf("%s: %s", - r.ServerInfo(), - pp.Sprintf("%s", c.Conf.Servers[r.ServerName])) + r.ServerInfo(), pp.Sprintf("%s", c.Conf.Servers[r.ServerName])) } // Ensure UUIDs of scan target servers in config.toml @@ -118,7 +112,7 @@ func (p *SaaSCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitFailure } - var w report.ResultWriter = saas.Writer{} + var w reporter.ResultWriter = saas.Writer{} if err := w.Write(res...); err != nil { util.Log.Errorf("Failed to upload. err: %+v", err) return subcommands.ExitFailure diff --git a/subcmds/scan.go b/subcmds/scan.go index 40bd15a0..e5b48a4d 100644 --- a/subcmds/scan.go +++ b/subcmds/scan.go @@ -9,8 +9,9 @@ import ( "path/filepath" "strings" + "github.com/asaskevich/govalidator" c "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/scan" + "github.com/future-architect/vuls/scanner" "github.com/future-architect/vuls/util" "github.com/google/subcommands" "github.com/k0kubun/pp" @@ -22,6 +23,7 @@ type ScanCmd struct { askKeyPassword bool timeoutSec int scanTimeoutSec int + cacheDBPath string } // Name return subcommand name @@ -38,7 +40,6 @@ func (*ScanCmd) Usage() string { [-results-dir=/path/to/results] [-log-dir=/path/to/log] [-cachedb-path=/path/to/cache.db] - [-ssh-native-insecure] [-http-proxy=http://192.168.0.1:8080] [-ask-key-password] [-timeout=300] @@ -70,12 +71,9 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) { f.StringVar(&c.Conf.LogDir, "log-dir", defaultLogDir, "/path/to/log") defaultCacheDBPath := filepath.Join(wd, "cache.db") - f.StringVar(&c.Conf.CacheDBPath, "cachedb-path", defaultCacheDBPath, + f.StringVar(&p.cacheDBPath, "cachedb-path", defaultCacheDBPath, "/path/to/cache.db (local cache of changelog for Ubuntu/Debian)") - f.BoolVar(&c.Conf.SSHNative, "ssh-native-insecure", false, - "Use Native Go implementation of SSH. Default: Use the external command") - f.StringVar(&c.Conf.HTTPProxy, "http-proxy", "", "http://proxy-url:port (default: empty)") @@ -103,10 +101,18 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) util.Log = util.NewCustomLogger(c.ServerInfo{}) if err := mkdirDotVuls(); err != nil { - util.Log.Errorf("Failed to create .vuls. err: %+v", err) + util.Log.Errorf("Failed to create $HOME/.vuls err: %+v", err) return subcommands.ExitUsageError } + if len(p.cacheDBPath) != 0 { + if ok, _ := govalidator.IsFilePath(p.cacheDBPath); !ok { + util.Log.Errorf("Cache DB path must be a *Absolute* file path. -cache-dbpath: %s", + p.cacheDBPath) + return subcommands.ExitUsageError + } + } + var keyPass string var err error if p.askKeyPassword { @@ -161,6 +167,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitUsageError } } + // if scan target servers are specified by args, set to the config if 0 < len(servernames) { c.Conf.Servers = target } @@ -171,27 +178,18 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitUsageError } - util.Log.Info("Detecting Server/Container OS... ") - if err := scan.InitServers(p.timeoutSec); err != nil { - util.Log.Errorf("Failed to init servers: %+v", err) + s := scanner.Scanner{ + TimeoutSec: p.timeoutSec, + ScanTimeoutSec: p.scanTimeoutSec, + CacheDBPath: p.cacheDBPath, + Targets: target, + } + + if err := s.Scan(); err != nil { + util.Log.Errorf("Failed to scan: %+v", err) return subcommands.ExitFailure } - util.Log.Info("Checking Scan Modes... ") - if err := scan.CheckScanModes(); err != nil { - util.Log.Errorf("Fix config.toml. err: %+v", err) - return subcommands.ExitFailure - } - - util.Log.Info("Detecting Platforms... ") - scan.DetectPlatforms(p.timeoutSec) - util.Log.Info("Detecting IPS identifiers... ") - scan.DetectIPSs(p.timeoutSec) - - if err := scan.Scan(p.scanTimeoutSec); err != nil { - util.Log.Errorf("Failed to scan. err: %+v", err) - return subcommands.ExitFailure - } fmt.Printf("\n\n\n") fmt.Println("To view the detail, vuls tui is useful.") fmt.Println("To send a report, run vuls report -h.") diff --git a/subcmds/server.go b/subcmds/server.go index 55670cd8..e64975f6 100644 --- a/subcmds/server.go +++ b/subcmds/server.go @@ -14,7 +14,7 @@ import ( "github.com/future-architect/vuls/config" c "github.com/future-architect/vuls/config" - "github.com/future-architect/vuls/report" + "github.com/future-architect/vuls/detector" "github.com/future-architect/vuls/server" "github.com/future-architect/vuls/util" "github.com/google/subcommands" @@ -22,8 +22,9 @@ import ( // ServerCmd is subcommand for server type ServerCmd struct { - configPath string - listen string + configPath string + listen string + toLocalFile bool } // Name return subcommand name @@ -43,7 +44,6 @@ func (*ServerCmd) Usage() string { [-ignore-unscored-cves] [-ignore-unfixed] [-to-localfile] - [-format-json] [-http-proxy=http://192.168.0.1:8080] [-debug] [-debug-sql] @@ -81,9 +81,7 @@ func (p *ServerCmd) SetFlags(f *flag.FlagSet) { f.StringVar(&c.Conf.HTTPProxy, "http-proxy", "", "http://proxy-url:port (default: empty)") - f.BoolVar(&c.Conf.FormatJSON, "format-json", false, "JSON format") - - f.BoolVar(&c.Conf.ToLocalFile, "to-localfile", false, "Write report to localfile") + f.BoolVar(&p.toLocalFile, "to-localfile", false, "Write report to localfile") f.StringVar(&p.listen, "listen", "localhost:5515", "host:port (default: localhost:5515)") } @@ -119,7 +117,7 @@ func (p *ServerCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } } - dbclient, locked, err := report.NewDBClient(report.DBClientConf{ + dbclient, locked, err := detector.NewDBClient(detector.DBClientConf{ CveDictCnf: c.Conf.CveDict, OvalDictCnf: c.Conf.OvalDict, GostCnf: c.Conf.Gost, @@ -139,7 +137,10 @@ func (p *ServerCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} defer dbclient.CloseDB() - http.Handle("/vuls", server.VulsHandler{DBclient: *dbclient}) + http.Handle("/vuls", server.VulsHandler{ + DBclient: *dbclient, + ToLocalFile: p.toLocalFile, + }) http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "ok") }) diff --git a/subcmds/tui.go b/subcmds/tui.go index 610bc2ac..2ae3fdaa 100644 --- a/subcmds/tui.go +++ b/subcmds/tui.go @@ -11,8 +11,10 @@ import ( "github.com/aquasecurity/trivy/pkg/utils" "github.com/future-architect/vuls/config" c "github.com/future-architect/vuls/config" + "github.com/future-architect/vuls/detector" "github.com/future-architect/vuls/models" - "github.com/future-architect/vuls/report" + "github.com/future-architect/vuls/reporter" + "github.com/future-architect/vuls/tui" "github.com/future-architect/vuls/util" "github.com/google/subcommands" ) @@ -115,9 +117,9 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s var dir string var err error if c.Conf.DiffPlus || c.Conf.DiffMinus { - dir, err = report.JSONDir([]string{}) + dir, err = reporter.JSONDir([]string{}) } else { - dir, err = report.JSONDir(f.Args()) + dir, err = reporter.JSONDir(f.Args()) } if err != nil { util.Log.Errorf("Failed to read from JSON. err: %+v", err) @@ -130,7 +132,7 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s } var res models.ScanResults - if res, err = report.LoadScanResults(dir); err != nil { + if res, err = reporter.LoadScanResults(dir); err != nil { util.Log.Error(err) return subcommands.ExitFailure } @@ -154,7 +156,7 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s } } - dbclient, locked, err := report.NewDBClient(report.DBClientConf{ + dbclient, locked, err := detector.NewDBClient(detector.DBClientConf{ CveDictCnf: c.Conf.CveDict, OvalDictCnf: c.Conf.OvalDict, GostCnf: c.Conf.Gost, @@ -174,7 +176,7 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s defer dbclient.CloseDB() - if res, err = report.FillCveInfos(*dbclient, res, dir); err != nil { + if res, err = detector.Detect(*dbclient, res, dir); err != nil { util.Log.Error(err) return subcommands.ExitFailure } @@ -186,5 +188,5 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s } } - return report.RunTui(res) + return tui.RunTui(res) } diff --git a/report/tui.go b/tui/tui.go similarity index 99% rename from report/tui.go rename to tui/tui.go index 0cece248..1d87331f 100644 --- a/report/tui.go +++ b/tui/tui.go @@ -1,4 +1,4 @@ -package report +package tui import ( "bytes" @@ -826,7 +826,7 @@ func setChangelogLayout(g *gocui.Gui) error { "=============", ) for _, alert := range vinfo.AlertDict.Ja { - if config.Conf.Lang == "ja" { + if r.Lang == "ja" { lines = append(lines, fmt.Sprintf("* [%s](%s)", alert.Title, alert.URL)) } else { lines = append(lines, fmt.Sprintf("* [JPCERT](%s)", alert.URL)) @@ -834,7 +834,7 @@ func setChangelogLayout(g *gocui.Gui) error { } } - if currentScanResult.IsDeepScanMode() { + if currentScanResult.Config.Scan.Servers[currentScanResult.ServerName].Mode.IsDeep() { lines = append(lines, "\n", "ChangeLogs", "==========", @@ -894,7 +894,7 @@ func detailLines() (string, error) { vinfo := vinfos[currentVinfo] links := []string{} - for _, r := range vinfo.CveContents.PrimarySrcURLs(config.Conf.Lang, r.Family, vinfo.CveID) { + for _, r := range vinfo.CveContents.PrimarySrcURLs(r.Lang, r.Family, vinfo.CveID) { links = append(links, r.Value) } @@ -934,7 +934,7 @@ func detailLines() (string, error) { } table := uitable.New() - table.MaxColWidth = maxColWidth + table.MaxColWidth = 100 table.Wrap = true scores := append(vinfo.Cvss3Scores(), vinfo.Cvss2Scores()...) var cols []interface{}