diff --git a/commands/report.go b/commands/report.go index c6dd9987..9c0c2077 100644 --- a/commands/report.go +++ b/commands/report.go @@ -235,85 +235,9 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} return subcommands.ExitFailure } - // report - reports := []report.ResultWriter{ - report.StdoutWriter{}, - } - - if c.Conf.ToSlack { - reports = append(reports, report.SlackWriter{}) - } - - if c.Conf.ToStride { - reports = append(reports, report.StrideWriter{}) - } - - if c.Conf.ToHipChat { - reports = append(reports, report.HipChatWriter{}) - } - - if c.Conf.ToChatWork { - reports = append(reports, report.ChatWorkWriter{}) - } - - if c.Conf.ToTelegram { - reports = append(reports, report.TelegramWriter{}) - } - - if c.Conf.ToEmail { - reports = append(reports, report.EMailWriter{}) - } - - if c.Conf.ToSyslog { - reports = append(reports, report.SyslogWriter{}) - } - - if c.Conf.ToHTTP { - reports = append(reports, report.HTTPRequestWriter{}) - } - - if c.Conf.ToLocalFile { - reports = append(reports, report.LocalFileWriter{ - CurrentDir: dir, - }) - } - - if c.Conf.ToS3 { - if err := report.CheckIfBucketExists(); err != nil { - util.Log.Errorf("Check if there is a bucket beforehand: %s, err: %+v", - c.Conf.AWS.S3Bucket, err) - return subcommands.ExitUsageError - } - reports = append(reports, report.S3Writer{}) - } - - if c.Conf.ToAzureBlob { - if len(c.Conf.Azure.AccountName) == 0 { - c.Conf.Azure.AccountName = os.Getenv("AZURE_STORAGE_ACCOUNT") - } - - if len(c.Conf.Azure.AccountKey) == 0 { - c.Conf.Azure.AccountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY") - } - - if len(c.Conf.Azure.ContainerName) == 0 { - util.Log.Error("Azure storage container name is required with -azure-container option") - return subcommands.ExitUsageError - } - if err := report.CheckIfAzureContainerExists(); err != nil { - util.Log.Errorf("Check if there is a container beforehand: %s, err: %+v", - c.Conf.Azure.ContainerName, err) - return subcommands.ExitUsageError - } - reports = append(reports, report.AzureBlobWriter{}) - } - - if c.Conf.ToSaas { - if !c.Conf.UUID { - util.Log.Errorf("If you use the -to-saas option, you need to enable the uuid option") - return subcommands.ExitUsageError - } - reports = append(reports, report.SaasWriter{}) + util.Log.Info("Validating config...") + if !c.Conf.ValidateOnReport() { + return subcommands.ExitUsageError } if !(c.Conf.FormatJSON || c.Conf.FormatOneLineText || @@ -321,11 +245,6 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} c.Conf.FormatList = true } - util.Log.Info("Validating config...") - if !c.Conf.ValidateOnReport() { - return subcommands.ExitUsageError - } - var loaded models.ScanResults if loaded, err = report.LoadScanResults(dir); err != nil { util.Log.Error(err) @@ -437,6 +356,87 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } } + // report + reports := []report.ResultWriter{ + report.StdoutWriter{}, + } + + if c.Conf.ToSlack { + reports = append(reports, report.SlackWriter{}) + } + + if c.Conf.ToStride { + reports = append(reports, report.StrideWriter{}) + } + + if c.Conf.ToHipChat { + reports = append(reports, report.HipChatWriter{}) + } + + if c.Conf.ToChatWork { + reports = append(reports, report.ChatWorkWriter{}) + } + + if c.Conf.ToTelegram { + reports = append(reports, report.TelegramWriter{}) + } + + if c.Conf.ToEmail { + reports = append(reports, report.EMailWriter{}) + } + + if c.Conf.ToSyslog { + reports = append(reports, report.SyslogWriter{}) + } + + if c.Conf.ToHTTP { + reports = append(reports, report.HTTPRequestWriter{}) + } + + if c.Conf.ToLocalFile { + reports = append(reports, report.LocalFileWriter{ + CurrentDir: dir, + }) + } + + if c.Conf.ToS3 { + if err := report.CheckIfBucketExists(); err != nil { + util.Log.Errorf("Check if there is a bucket beforehand: %s, err: %+v", + c.Conf.AWS.S3Bucket, err) + return subcommands.ExitUsageError + } + reports = append(reports, report.S3Writer{}) + } + + if c.Conf.ToAzureBlob { + if len(c.Conf.Azure.AccountName) == 0 { + c.Conf.Azure.AccountName = os.Getenv("AZURE_STORAGE_ACCOUNT") + } + + if len(c.Conf.Azure.AccountKey) == 0 { + c.Conf.Azure.AccountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY") + } + + if len(c.Conf.Azure.ContainerName) == 0 { + util.Log.Error("Azure storage container name is required with -azure-container option") + return subcommands.ExitUsageError + } + if err := report.CheckIfAzureContainerExists(); err != nil { + util.Log.Errorf("Check if there is a container beforehand: %s, err: %+v", + c.Conf.Azure.ContainerName, err) + return subcommands.ExitUsageError + } + reports = append(reports, report.AzureBlobWriter{}) + } + + if c.Conf.ToSaas { + if !c.Conf.UUID { + util.Log.Errorf("If you use the -to-saas option, you need to enable the uuid option") + return subcommands.ExitUsageError + } + reports = append(reports, report.SaasWriter{}) + } + for _, w := range reports { if err := w.Write(res...); err != nil { util.Log.Errorf("Failed to report. err: %+v", err) diff --git a/config/config.go b/config/config.go index 8361fd21..a4770e53 100644 --- a/config/config.go +++ b/config/config.go @@ -1121,18 +1121,17 @@ type ServerInfo struct { Lockfiles []string `toml:"lockfiles,omitempty" json:"lockfiles,omitempty"` // ie) path/to/package-lock.json FindLock bool `toml:"findLock,omitempty" json:"findLock,omitempty"` Type string `toml:"type,omitempty" json:"type,omitempty"` // "pseudo" or "" + WordPress WordPressConf `toml:"wordpress,omitempty" json:"wordpress,omitempty"` + IgnoredJSONKeys []string `toml:"ignoredJSONKeys,omitempty" json:"ignoredJSONKeys,omitempty"` - WordPress WordPressConf `toml:"wordpress,omitempty" json:"wordpress,omitempty"` - - // used internal - IPv4Addrs []string `toml:"-" json:"ipv4Addrs,omitempty"` - IPv6Addrs []string `toml:"-" json:"ipv6Addrs,omitempty"` - IPSIdentifiers map[IPS]string `toml:"-" json:"ipsIdentifiers,omitempty"` - - LogMsgAnsiColor string `toml:"-" json:"-"` // DebugLog Color - Container Container `toml:"-" json:"-"` - Distro Distro `toml:"-" json:"-"` - Mode ScanMode `toml:"-" json:"-"` + // internal use + IPv4Addrs []string `toml:"-" json:"ipv4Addrs,omitempty"` + IPv6Addrs []string `toml:"-" json:"ipv6Addrs,omitempty"` + IPSIdentifiers map[IPS]string `toml:"-" json:"ipsIdentifiers,omitempty"` + LogMsgAnsiColor string `toml:"-" json:"-"` // DebugLog Color + Container Container `toml:"-" json:"-"` + Distro Distro `toml:"-" json:"-"` + Mode ScanMode `toml:"-" json:"-"` } // ContainerSetting is used for loading container setting in config.toml diff --git a/config/tomlloader.go b/config/tomlloader.go index 5c30ae8f..b772fd9e 100644 --- a/config/tomlloader.go +++ b/config/tomlloader.go @@ -45,7 +45,7 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error { d.KeyPassword = keyPass } - i := 0 + index := 0 for serverName, v := range conf.Servers { if 0 < len(v.KeyPassword) { return xerrors.Errorf("[Deprecated] KEYPASSWORD IN CONFIG FILE ARE UNSECURE. REMOVE THEM IMMEDIATELY FOR A SECURITY REASONS. THEY WILL BE REMOVED IN A FUTURE RELEASE: %s", serverName) @@ -268,8 +268,13 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error { s.WordPress.OSUser = v.WordPress.OSUser s.WordPress.IgnoreInactive = v.WordPress.IgnoreInactive - s.LogMsgAnsiColor = Colors[i%len(Colors)] - i++ + s.IgnoredJSONKeys = v.IgnoredJSONKeys + if len(s.IgnoredJSONKeys) == 0 { + s.IgnoredJSONKeys = d.IgnoredJSONKeys + } + + s.LogMsgAnsiColor = Colors[index%len(Colors)] + index++ servers[serverName] = s } diff --git a/models/scanresults.go b/models/scanresults.go index 59517f2d..dbc4512f 100644 --- a/models/scanresults.go +++ b/models/scanresults.go @@ -3,6 +3,7 @@ package models import ( "bytes" "fmt" + "reflect" "regexp" "strings" "time" @@ -503,3 +504,23 @@ func (r ScanResult) RemoveRaspbianPackFromResult() ScanResult { return result } + +func (r ScanResult) ClearFields(targetTagNames []string) ScanResult { + if len(targetTagNames) == 0 { + return r + } + target := map[string]bool{} + for _, n := range targetTagNames { + target[strings.ToLower(n)] = true + } + t := reflect.ValueOf(r).Type() + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + jsonValue := strings.Split(f.Tag.Get("json"), ",")[0] + if ok := target[strings.ToLower(jsonValue)]; ok { + vv := reflect.New(f.Type).Elem().Interface() + reflect.ValueOf(&r).Elem().FieldByName(f.Name).Set(reflect.ValueOf(vv)) + } + } + return r +} diff --git a/report/localfile.go b/report/localfile.go index e0a95579..43cea153 100644 --- a/report/localfile.go +++ b/report/localfile.go @@ -41,14 +41,8 @@ func (w LocalFileWriter) Write(rs ...models.ScanResult) (err error) { } var b []byte - if c.Conf.Debug { - if b, err = json.MarshalIndent(r, "", " "); err != nil { - return xerrors.Errorf("Failed to Marshal to JSON: %w", err) - } - } else { - if b, err = json.Marshal(r); err != nil { - return xerrors.Errorf("Failed to Marshal to JSON: %w", err) - } + if b, err = json.MarshalIndent(r, "", " "); err != nil { + return xerrors.Errorf("Failed to Marshal to JSON: %w", err) } if err := writeFile(p, b, 0600); err != nil { return xerrors.Errorf("Failed to write JSON. path: %s, err: %w", p, err) diff --git a/report/report.go b/report/report.go index e194c4f0..4913243f 100644 --- a/report/report.go +++ b/report/report.go @@ -43,105 +43,112 @@ const ( // FillCveInfos fills CVE Detailed Information func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.ScanResult, error) { - var filledResults []models.ScanResult + + // Use the same reportedAt for all rs reportedAt := time.Now() - hostname, _ := os.Hostname() - wpVulnCaches := map[string]string{} - for _, r := range rs { - if c.Conf.RefreshCve || needToRefreshCve(r) { - if !useScannedCves(&r) { - r.ScannedCves = models.VulnInfos{} - } - cpeURIs := []string{} + for i, r := range rs { + if !c.Conf.RefreshCve && !needToRefreshCve(r) { + util.Log.Info("No need to refresh") + continue + } - if len(r.Container.ContainerID) == 0 { - cpeURIs = c.Conf.Servers[r.ServerName].CpeNames - owaspDCXMLPath := c.Conf.Servers[r.ServerName].OwaspDCXMLPath - if owaspDCXMLPath != "" { - cpes, err := parser.Parse(owaspDCXMLPath) - if err != nil { - return nil, xerrors.Errorf("Failed to read OWASP Dependency Check XML on %s, `%s`, err: %w", - r.ServerName, owaspDCXMLPath, err) - } - cpeURIs = append(cpeURIs, cpes...) - } - } else { - // runningContainer - if s, ok := c.Conf.Servers[r.ServerName]; ok { - if con, ok := s.Containers[r.Container.Name]; ok { - cpeURIs = con.Cpes - owaspDCXMLPath := con.OwaspDCXMLPath - if owaspDCXMLPath != "" { - cpes, err := parser.Parse(owaspDCXMLPath) - if err != nil { - return nil, xerrors.Errorf("Failed to read OWASP Dependency Check XML on %s, `%s`, err: %w", - r.ServerInfo(), owaspDCXMLPath, err) - } - cpeURIs = append(cpeURIs, cpes...) - } - } + if !useScannedCves(&r) { + r.ScannedCves = models.VulnInfos{} + } + + cpeURIs := []string{} + if len(r.Container.ContainerID) == 0 { + cpeURIs = c.Conf.Servers[r.ServerName].CpeNames + owaspDCXMLPath := c.Conf.Servers[r.ServerName].OwaspDCXMLPath + if owaspDCXMLPath != "" { + cpes, err := parser.Parse(owaspDCXMLPath) + if err != nil { + return nil, xerrors.Errorf("Failed to read OWASP Dependency Check XML on %s, `%s`, err: %w", + r.ServerName, owaspDCXMLPath, err) } + cpeURIs = append(cpeURIs, cpes...) } - - nCVEs, err := libmanager.DetectLibsCves(&r) - if err != nil { - return nil, xerrors.Errorf("Failed to fill with Library dependency: %w", err) - } - util.Log.Infof("%s: %d CVEs are detected with Library", - r.FormatServerName(), nCVEs) - - // Integrations - githubInts := GithubSecurityAlerts(c.Conf.Servers[r.ServerName].GitHubRepos) - wpOpt := WordPressOption{c.Conf.Servers[r.ServerName].WordPress.WPVulnDBToken, &wpVulnCaches} - - if err := FillCveInfo(dbclient, - &r, - cpeURIs, - true, - githubInts, - wpOpt); err != nil { - return nil, err - } - r.Lang = c.Conf.Lang - r.ReportedAt = reportedAt - r.ReportedVersion = c.Version - r.ReportedRevision = c.Revision - r.ReportedBy = hostname - r.Config.Report = c.Conf - r.Config.Report.Servers = map[string]c.ServerInfo{ - r.ServerName: c.Conf.Servers[r.ServerName], - } - if err := overwriteJSONFile(dir, r); err != nil { - return nil, xerrors.Errorf("Failed to write JSON: %w", err) - } - filledResults = append(filledResults, r) } else { - util.Log.Debugf("No need to refresh") - filledResults = append(filledResults, r) + // runningContainer + if s, ok := c.Conf.Servers[r.ServerName]; ok { + if con, ok := s.Containers[r.Container.Name]; ok { + cpeURIs = con.Cpes + owaspDCXMLPath := con.OwaspDCXMLPath + if owaspDCXMLPath != "" { + cpes, err := parser.Parse(owaspDCXMLPath) + if err != nil { + return nil, xerrors.Errorf("Failed to read OWASP Dependency Check XML on %s, `%s`, err: %w", + r.ServerInfo(), owaspDCXMLPath, err) + } + cpeURIs = append(cpeURIs, cpes...) + } + } + } + } + + nCVEs, err := libmanager.DetectLibsCves(&r) + if err != nil { + return nil, xerrors.Errorf("Failed to fill with Library dependency: %w", err) + } + util.Log.Infof("%s: %d CVEs are detected with Library", + r.FormatServerName(), nCVEs) + + // Integrations + githubInts := GithubSecurityAlerts(c.Conf.Servers[r.ServerName].GitHubRepos) + + wpVulnCaches := map[string]string{} + wpOpt := WordPressOption{c.Conf.Servers[r.ServerName].WordPress.WPVulnDBToken, &wpVulnCaches} + + if err := FillCveInfo(dbclient, + &r, + cpeURIs, + true, + githubInts, + wpOpt); err != nil { + return nil, err + } + + r.ReportedBy, _ = os.Hostname() + r.Lang = c.Conf.Lang + r.ReportedAt = reportedAt + r.ReportedVersion = c.Version + r.ReportedRevision = c.Revision + r.Config.Report = c.Conf + r.Config.Report.Servers = map[string]c.ServerInfo{ + r.ServerName: c.Conf.Servers[r.ServerName], + } + rs[i] = r + } + + // Overwrite the json file every time to clear the fields specified in config.IgnoredJSONKeys + for _, r := range rs { + if s, ok := c.Conf.Servers[r.ServerName]; ok { + r = r.ClearFields(s.IgnoredJSONKeys) + } + if err := overwriteJSONFile(dir, r); err != nil { + return nil, xerrors.Errorf("Failed to write JSON: %w", err) } } if c.Conf.Diff { - prevs, err := loadPrevious(filledResults) + prevs, err := loadPrevious(rs) if err != nil { return nil, err } - diff, err := diff(filledResults, prevs) + diff, err := diff(rs, prevs) if err != nil { return nil, err } - filledResults = []models.ScanResult{} - for _, r := range diff { + for i, r := range diff { if err := fillCvesWithNvdJvn(dbclient.CveDB, &r); err != nil { return nil, err } - filledResults = append(filledResults, r) + rs[i] = r } } - filtered := []models.ScanResult{} - for _, r := range filledResults { + for i, r := range rs { r = r.FilterByCvssOver(c.Conf.CvssScoreOver) r = r.FilterIgnoreCves() r = r.FilterUnfixed() @@ -150,9 +157,9 @@ func FillCveInfos(dbclient DBClient, rs []models.ScanResult, dir string) ([]mode if c.Conf.IgnoreUnscoredCves { r.ScannedCves = r.ScannedCves.FindScoredVulns() } - filtered = append(filtered, r) + rs[i] = r } - return filtered, nil + return rs, nil } // FillCveInfo fill scanResult with cve info.